[2024-12-02 13:55:05.774540] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF594-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.839351] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC6-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:05.862018] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:05.875454] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.875465] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:05.875480] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:05.875489] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:05.878784] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF595-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.940075] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC37-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:05.952073] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:05.980397] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:05.981959] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:05.981970] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:05.981985] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:05.981994] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF24F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:05.982230] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF596-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.026960] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAD-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:06.030444] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC2-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.032185] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAD-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:06.051978] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC3F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.085861] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF597-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.086893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.088433] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.088443] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.088458] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.088467] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF250-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.110178] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.112729] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:06.112750] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC25-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.132725] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.134100] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:06.134155] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:06.134166] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:06.134177] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:06.134185] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:06.134197] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:06.134204] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118911132741, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:06.189118] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF598-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.193349] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.194993] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA94-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.195219] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.195230] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.195246] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.195255] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF251-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.209028] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.210185] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906208962, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:06.210213] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911210211, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:06.235763] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC43-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.293296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF599-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.300145] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.301511] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.301521] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.301536] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.301545] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF252-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.303947] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:06.305099] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906303883, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:06.305126] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911305123, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:06.306253] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118906304998, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:06.306279] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911306277, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:06.312083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA94-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.352006] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:06.352025] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=260, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:06.353311] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:06.397056] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.406434] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.407968] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.407979] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.407995] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.408001] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF253-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.480586] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC46-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:06.500891] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59B-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.512892] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.514099] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.514109] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.514125] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.514134] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF254-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.535175] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.536335] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:361) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=18] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-12-02 13:55:06.538800] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC26-0-0] [lt=10] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:06.558879] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7B-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.561274] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7B-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:06.604384] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.618940] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.620500] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.620510] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.620526] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.620535] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF255-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.708058] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59D-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.725414] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.726611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.726622] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.726637] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.726646] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF256-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.759042] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE965-0-0] [lt=31] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:06.811616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59E-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.831580] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.832862] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.832872] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.832886] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.832894] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF257-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.915288] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.931205] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF59F-0-0] [lt=52] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.932560] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=28] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:06.933784] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=15] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:06.934969] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE965-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.937764] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:06.938987] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:06.938994] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:06.939008] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:06.939016] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF258-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:06.941414] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC38-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.031530] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A0-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.043703] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118906703958001, v:0}}], last_dump_ts_=1733118901959829, force_dump=false) [2024-12-02 13:55:07.043730] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-02 13:55:07.043937] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.045168] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.045178] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.045193] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.045200] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF259-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.048844] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAE-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:55:07.052056] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC40-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.115438] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:07.117971] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:07.117989] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC26-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:07.135181] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.150023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.151661] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.151671] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.151686] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.151693] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.195069] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA95-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.235859] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC44-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.238917] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.256548] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.257865] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.257875] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.257890] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.257898] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.312158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA95-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.342276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.351868] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118907348911, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118907348911, ora_rowscn_=1733118907045839000, content_changed=true) [2024-12-02 13:55:07.352080] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:07.362769] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.364585] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.364594] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.364608] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.364617] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.403110] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2024-12-02 13:55:07.404899] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118907348911, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118905555272004, replayable_scn:1733118905555272004, readable_scn:1733118905555272003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:07.445707] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A4-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.469556] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.470773] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.470783] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.470797] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.470805] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.480660] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC47-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.544049] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC27-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:07.546464] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC27-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.549270] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A5-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.552325] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A5-0-0] [lt=52] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.568030] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7C-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:07.570477] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7C-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.575623] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.577160] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.577172] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.577189] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.577199] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.620440] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC1-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-02 13:55:07.652791] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.682076] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.683334] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.683345] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:07.683361] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:07.683370] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF25F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.756570] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A7-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.788180] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF260-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.789649] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF260-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.844117] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC7-0-0] [lt=24] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:07.851796] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118907848637, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118907848637, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:07.860174] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:07.894387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF261-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:07.895811] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF261-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:07.920783] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=19] mview maintenace task finish(tenant_id_=1, round_=10, status_=100, error_code_=0, cost_us_=20001259, prepare_cost_us_=0, gc_mview_cost_us_=1095, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:55:07.939416] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:07.942772] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC39-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:07.965056] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95C-0-0] [lt=10] start do ObArchiveSchedulerService round(round=17) [2024-12-02 13:55:07.967273] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95C-0-0] [lt=19][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:07.970062] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2319871][T1_BackupDataSr][T1][YB42C0A800AF-0006284308DEE966-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:07.976737] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5A9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.000656] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF262-0-0] [lt=25] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.002128] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF262-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.030752] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC3-0-0] [lt=11] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.052130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC41-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.053536] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAF-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:08.058587] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECAF-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:08.080157] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.106848] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=26] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.108661] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.108672] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.108688] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.108697] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF263-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.120623] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:08.123147] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:08.123167] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC27-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:08.183490] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AB-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.195147] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA96-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.213597] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.214847] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.214857] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.214873] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.214882] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF264-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.235962] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC45-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.287276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.312188] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA96-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.319881] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.321142] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.321154] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.321170] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.321179] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF265-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.352177] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:08.390719] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AD-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.426072] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=31] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.427291] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.427302] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.427318] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.427327] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF266-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.480735] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC48-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:08.494370] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.494621] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AE-0-0] [lt=51] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118903471521, force_dump=false) [2024-12-02 13:55:08.532207] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.551884] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC28-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:08.554351] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC28-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.577353] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7D-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:08.577970] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.577982] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.577998] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.578008] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF267-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.579755] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7D-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.672348] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5AF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.682844] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.684298] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.684307] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.684319] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.684327] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF268-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.776024] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B0-0-0] [lt=50] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118903678547, force_dump=false) [2024-12-02 13:55:08.776058] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B0-0-0] [lt=8][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.789108] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.796782] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.796793] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.796809] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.796819] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF269-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.879641] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B1-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.901547] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:08.903139] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:08.903150] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:08.903167] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:08.903172] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26A-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:08.944160] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3A-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:08.995380] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.008021] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.009233] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=14][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.009240] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.009256] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.009265] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.052202] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC42-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.063767] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:09.069598] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=20] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118908558064003, v:0}}], last_dump_ts_=1733118904009593, force_dump=false) [2024-12-02 13:55:09.069621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB0-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:09.098665] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.114023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.121639] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.121650] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.121665] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.121671] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.125872] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:09.128384] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:09.128402] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC28-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:09.195225] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA97-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.201969] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.226428] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.228751] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.228761] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.228776] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.228786] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.236074] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC46-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.283918] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE997-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733118909283081000, start_id=1733118909283081000, end_id=1733118909283081001) [2024-12-02 13:55:09.285668] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE997-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118909283081000, v:0}, tenant_id=1) [2024-12-02 13:55:09.312259] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA97-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.313956] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.333584] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.334805] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.334815] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.334830] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.334835] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26E-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.352242] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:09.417364] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B6-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.439712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.441214] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.441225] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.441241] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.441250] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF26F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.480788] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC49-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.520873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.545989] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.547201] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.547212] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.547227] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.547236] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF270-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.559697] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC29-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:09.562197] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC29-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.586605] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7E-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:09.589049] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.620868] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC2-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-02 13:55:09.624232] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.651959] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.653448] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.653459] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.653482] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=22][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.653488] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF271-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.728083] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5B9-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.758278] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.759822] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.759832] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.759848] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.759857] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF272-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.831706] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.848969] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC8-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:09.864624] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.873116] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.873127] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.873142] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.873151] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF273-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:09.935503] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BB-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.945556] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3B-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:09.977953] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:09.979468] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:09.979479] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:09.979494] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:09.979500] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF274-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.031064] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC4-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.038885] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.052263] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC43-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.075072] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB1-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:10.080885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB1-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:10.084340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.085587] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.085597] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.085613] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.085619] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF275-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.131095] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:10.133521] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:10.133539] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC29-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:10.142460] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BD-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.190550] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.192114] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=10][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:10.192129] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF276-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.195302] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA98-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.236173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC47-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.246038] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.296843] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.298176] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.298186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.298202] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.298207] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF277-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.312331] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA98-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.349636] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5BF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.352096] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118910348801, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118910348801, ora_rowscn_=1733118910077385000, content_changed=true) [2024-12-02 13:55:10.352320] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=26] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:10.352337] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=213, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:10.403059] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.404660] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.404671] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.404687] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.404697] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF278-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.406515] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118910348801, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118908558064004, replayable_scn:1733118908558064004, readable_scn:1733118908558064003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:10.454170] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.480863] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4A-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:10.509603] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.510827] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.510834] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.510851] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.510856] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF279-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.527278] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=10] start do ObArchiveSchedulerService round(round=17) [2024-12-02 13:55:10.530597] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:55:10.530614] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-12-02 13:55:10.530672] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:10.530689] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=16][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:55:10.530701] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95C-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:10.559251] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.567475] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2A-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:10.569909] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2A-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.595871] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7F-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:10.598338] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA7F-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.615766] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.617400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.617412] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.617428] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.617437] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.663055] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.722133] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.723392] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=14][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.723402] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.723418] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.723427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.766731] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.828236] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.829736] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.829747] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.829763] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.829772] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.853414] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118910850718, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118910850718, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:10.870409] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.934550] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:10.935781] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:10.935792] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:10.935808] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:10.935816] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:10.946949] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3C-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:10.954715] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:10.973978] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.040778] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.042392] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.042403] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.042420] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.042426] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.052339] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC44-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.077391] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.085793] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB2-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:55:11.091061] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB2-0-0] [lt=31] is primary tenant, no need check readable_scn [2024-12-02 13:55:11.132807] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.134226] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:11.134283] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:11.134294] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:11.134305] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:11.134317] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:11.134328] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:11.134336] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118916132825, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:11.136233] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:11.138792] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:11.138811] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2A-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:11.147233] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.148495] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.148506] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.148522] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.148531] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF27F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.182031] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.195376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA99-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.210276] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.211434] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911210211, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:11.211457] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916211455, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:11.214515] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96D-0-0] [lt=44] get number(ret=0, service_type_=0, range=1, base_id=1733118911213644000, start_id=1733118911213644000, end_id=1733118911213644001) [2024-12-02 13:55:11.216181] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96D-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118911213644000, v:0}, tenant_id=1019) [2024-12-02 13:55:11.236273] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC48-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.253446] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.254769] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.254780] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.254796] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.254805] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF280-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.285347] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.305187] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:11.306342] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911305123, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:11.306370] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916306369, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:11.307481] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118911306277, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:11.307505] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916307503, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:11.312402] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA99-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.352334] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:11.353677] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:11.359628] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.361095] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.361102] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.361118] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.361123] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF281-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.388559] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5C9-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.465944] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.467453] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.467464] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.467479] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.467489] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF282-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.480940] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4B-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.492276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.572342] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.573948] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.573959] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.573975] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.573984] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF283-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.575292] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2B-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:11.577771] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2B-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:11.595660] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CB-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.605084] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA80-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:11.607529] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA80-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:11.621302] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC3-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:11.678704] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.681011] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.681022] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.681038] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.681047] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF284-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.699626] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CC-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.785755] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.803005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.804949] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.804959] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.804975] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.804984] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF285-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.853775] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAC9-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:11.906924] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.909779] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:11.911356] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:11.911366] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:11.911382] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:11.911392] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF286-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:11.948318] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3D-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.010506] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5CF-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.016173] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.018224] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.018236] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.018249] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.018258] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF287-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.031376] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC5-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.052414] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC45-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.096452] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=14] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118911708100002, v:0}}], last_dump_ts_=1733118907043702, force_dump=false) [2024-12-02 13:55:12.096475] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:12.101765] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB3-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:12.114216] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.122955] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.124183] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.124193] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.124209] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.124218] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF288-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.141556] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:12.144110] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:12.144130] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2B-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:12.195453] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9A-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.218546] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.228919] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.230116] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.230127] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.230143] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.230152] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF289-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.236364] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC49-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.312474] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.322100] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.334903] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.336099] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.336106] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.336122] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.336131] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.352322] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:12.425619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.440803] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.442054] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.442064] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.442076] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.442084] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.481017] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4C-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:12.529285] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.546818] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.554150] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.554161] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.554177] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.554185] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.583198] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2C-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:12.585674] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2C-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.614304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA81-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:12.616702] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA81-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.632682] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D5-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.658974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.660312] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.660323] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.660338] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.660347] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.736287] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.765219] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.766535] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.766546] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.766563] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.766572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28E-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.840443] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D7-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.871663] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.873175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.873186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.873202] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.873211] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF28F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:12.944098] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.949724] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3E-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:12.975547] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=20] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=360, Holding Memory(MB)=61, Tenant Remain Memory(MB)=497, Usable Remain Memory(MB)=298, Last Update Limit Timestamp=1733118912975018, Is Updated=true) [2024-12-02 13:55:12.975718] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1733118912975018000, start_id=1733118912975018000, end_id=1733118912975018001) [2024-12-02 13:55:12.978282] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:12.978583] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE997-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118912975018000, v:0}, tenant_id=1001) [2024-12-02 13:55:12.980520] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:12.980531] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:12.980547] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:12.980557] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF290-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.047860] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5D9-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.052489] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC46-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.085540] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.087288] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.087298] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.087314] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.087324] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF291-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.112934] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB4-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:13.119503] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB4-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:13.146825] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:13.149351] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:13.149371] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2C-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:13.151991] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DA-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.192297] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.193928] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.193939] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.193955] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.193964] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF292-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.195527] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9B-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.236458] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4A-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.255755] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.281309] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC4-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:13.286176] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320107][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC5-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:13.288930] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDC9-0-0] [lt=12] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:13.289138] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCA-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:55:13.289760] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCB-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:13.289949] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCC-0-0] [lt=11] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:55:13.290140] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDCD-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:13.290882] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD3-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:13.291426] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD4-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:13.295171] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=19] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:13.299390] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.301237] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.301249] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.301267] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.301276] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF293-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.312546] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.352303] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118913348583, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118913348583, ora_rowscn_=1733118913114919001, content_changed=true) [2024-12-02 13:55:13.352584] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:13.359619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.387398] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=32] mview maintenace task finish(tenant_id_=1002, round_=10, status_=100, error_code_=0, cost_us_=20001321, prepare_cost_us_=1, gc_mview_cost_us_=1157, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:55:13.406441] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.408189] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=23] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118913348583, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118911708100002, replayable_scn:1733118911708100002, readable_scn:1733118911708100002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:13.408934] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.408945] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.408963] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.408972] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF294-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.463462] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DD-0-0] [lt=19][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:13.468265] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD6-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:13.481095] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4D-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.513900] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.515794] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.515804] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.515817] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.515825] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF295-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.566167] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5DE-0-0] [lt=0] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118908494621, force_dump=false) [2024-12-02 13:55:13.591042] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2D-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:13.593497] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2D-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:13.620828] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.621779] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC4-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-02 13:55:13.622469] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.622479] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.622496] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.622505] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF296-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.623363] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA82-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:13.626684] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA82-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:13.685386] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC1F-0-0] [lt=19] root balance idle(idle_time=10000000) [2024-12-02 13:55:13.685415] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=20] start try disaster recovery [2024-12-02 13:55:13.685427] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:13.686677] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.691243] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.691264] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691271] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.691322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=47] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:13.691331] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.691341] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.691373] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691385] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691392] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.691396] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=3] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.693356] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:13.693367] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:13.694654] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.699111] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.699132] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=16] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699139] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.699188] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=46] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:13.699197] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.699207] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.699239] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699251] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699257] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.699262] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.701045] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=3] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:13.702360] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:13.703584] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.708006] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.708036] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708047] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.708100] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:13.708113] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.708126] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.708160] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708175] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708184] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.708192] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.710167] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.716952] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:13.716982] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.716993] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.717045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:13.717058] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.717070] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.717104] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717118] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717126] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.717134] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.719065] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:13.719085] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:13.720331] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.724651] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.724678] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724687] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.724738] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=47] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:13.724749] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.724761] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.724794] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724811] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724818] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.724825] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.727116] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:13.727699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.728298] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:13.729351] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.729362] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.729378] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.729387] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF297-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.729484] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.733852] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:13.733882] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.733893] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.733946] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:13.733958] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.733971] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.734006] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734021] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734030] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.734039] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.735922] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:13.742646] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:13.742676] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742687] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:13.742739] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:13.742748] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:13.742761] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:13.742793] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742808] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742817] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.742825] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:13.744674] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:13.744692] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=15] start check unit migrate finish [2024-12-02 13:55:13.744706] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:13.750573] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:13.760902] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=18] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:13.771035] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC20-0-0] [lt=19] start to check alter locality finish [2024-12-02 13:55:13.772157] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC21-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.773257] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC21-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.775566] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC23-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.776669] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC23-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.778927] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.780047] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.781250] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC25-0-0] [lt=18] start check shrink resource pool [2024-12-02 13:55:13.781271] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:13.782340] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.783437] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC26-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.783515] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:13.784573] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.784627] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E0-0-0] [lt=1] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118908776022, force_dump=false) [2024-12-02 13:55:13.785688] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC27-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.785764] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=16] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:13.786815] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.787913] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.787990] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=17] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.788006] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC28-0-0] [lt=14] start check primary zone [2024-12-02 13:55:13.788018] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC29-0-0] [lt=7] start check primary zone(tenant_id=1) [2024-12-02 13:55:13.789056] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC29-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.789136] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2A-0-0] [lt=17] start check primary zone(tenant_id=1002) [2024-12-02 13:55:13.790194] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.790271] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=16] start check primary zone(tenant_id=1020) [2024-12-02 13:55:13.791306] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:13.791384] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] start do unit balance [2024-12-02 13:55:13.793836] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13] start distribute for server status change [2024-12-02 13:55:13.793889] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] finish distribute for server status change(ret=0) [2024-12-02 13:55:13.793899] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:55:13.796211] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=7] start to do tenant group unit balance [2024-12-02 13:55:13.796259] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=17] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:13.796276] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=11] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:13.796295] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:13.796319] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796339] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796350] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:13.796361] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.796384] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=16] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:13.796422] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=37] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:13.796432] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:13.796447] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:13.796459] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=10] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:13.834422] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=23] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.836120] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.836131] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.836147] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.836155] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF298-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.855096] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118913852503, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118913852503, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:13.858521] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACA-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:13.941110] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:13.943058] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:13.943068] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:13.943084] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:13.943093] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF299-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:13.951139] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC3F-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.031673] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC6-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.048139] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.049899] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.049908] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.049923] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.049931] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.052565] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC47-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.125619] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:14.133894] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=18] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118914077648001, v:0}}], last_dump_ts_=1733118909069596, force_dump=false) [2024-12-02 13:55:14.133916] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB5-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:14.138771] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.152066] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:14.154623] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:14.154643] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2D-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:14.154970] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.155928] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.155936] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.155951] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.155960] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.195599] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.198247] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=5] need remove task(remove_task_ids=[]) [2024-12-02 13:55:14.198264] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=16] remove all timeout ddl task succeed [2024-12-02 13:55:14.236557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4B-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.242389] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.260992] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.262611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.262621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.262635] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.262642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.312626] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9C-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.346090] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E5-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.352443] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:14.352459] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=241, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:14.367638] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.369594] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.369601] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.369616] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.369625] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.474595] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.475877] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.475887] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.475902] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.475907] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29E-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.481171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:14.556330] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5E7-0-0] [lt=17][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:14.580884] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.583372] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.583383] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.583395] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.583404] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF29F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.598859] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2E-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:14.601300] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.633277] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA83-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:14.635670] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA83-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:14.688277] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.690017] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.690028] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.690044] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.690053] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.794974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.796309] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.796319] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.796335] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.796344] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.901223] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:14.903138] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:14.903149] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:14.903165] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:14.903170] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:14.952485] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC40-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.008037] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.009251] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.009262] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.009277] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.009286] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.052643] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC48-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.114203] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.115789] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.115800] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.115816] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.115825] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.139539] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB6-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:55:15.149850] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB6-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:15.157360] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:15.159921] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:15.159940] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2E-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:15.179056] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5ED-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.195674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9D-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.220692] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.222302] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.222312] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.222329] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.222334] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A5-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.230369] INFO [SHARE] update_decay_factor_ (ob_throttle_unit.ipp:284) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=18] [Throttle] Update Throttle Unit Config(is_adaptive_update=true, N=5.980047273635864258e+01, this=0xffff2773b208, enable_adaptive_limit_=true, Unit Name=TxShare, Config Specify Resource Limit(MB)=511, Resource Limit(MB)=299, Throttle Trigger(MB)=179, Throttle Percentage=60, Max Duration(us)=7200000000, Decay Factor=2.178559058567542252e+03) [2024-12-02 13:55:15.230562] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=32] get number(ret=0, service_type_=0, range=1, base_id=1733118915229908000, start_id=1733118915229908000, end_id=1733118915229908001) [2024-12-02 13:55:15.232617] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE997-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118915229908000, v:0}, tenant_id=1002) [2024-12-02 13:55:15.236654] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4C-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.283188] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5EE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.312702] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9D-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.327338] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.352493] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:15.360411] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.360421] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.360436] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.360444] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A6-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.386549] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5EF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.465432] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.475897] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.475907] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.475923] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.475931] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A7-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.481246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC4F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.490203] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F0-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.580942] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.582545] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.582556] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.582568] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.582573] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A8-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.593599] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F1-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.606514] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2F-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:15.608925] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC2F-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.622192] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAC5-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:15.642377] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA84-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:15.644782] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA84-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.687502] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.689027] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.689038] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.689055] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.689064] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2A9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.708684] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.794285] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.795700] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.795712] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.795729] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.795740] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AA-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.812068] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.863192] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACB-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:15.900792] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:15.903197] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.903208] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:15.903223] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:15.903232] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:15.915781] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F4-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:15.953832] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC41-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:15.957407] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:16.008117] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.009307] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.009317] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.009333] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.009342] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.025866] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.031980] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC7-0-0] [lt=18] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.052725] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC49-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.114340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.116512] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.116519] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.116535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.116544] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.129271] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F6-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.132892] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.134285] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:16.134337] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:16.134348] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:16.134359] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:16.134367] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:16.134378] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:16.134386] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118921132911, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:16.154769] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:16.162579] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.165073] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:16.165093] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC2F-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.175199] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:16.195750] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9E-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.211518] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.212674] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916211455, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:16.212700] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921212698, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:16.221443] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.223101] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.223112] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.223128] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.223133] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.232961] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.236759] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4D-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.306433] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:16.307564] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916306369, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:16.307592] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921307590, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:16.308679] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118916307503, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:16.308697] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921308695, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:16.312775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9E-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.328079] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.329331] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.329341] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.329357] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.329365] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2AF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.337043] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F8-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.352359] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118916349261, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118916349261, ora_rowscn_=1733118916156831001, content_changed=true) [2024-12-02 13:55:16.352608] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:16.353945] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:16.409820] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118916349261, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118915064689000, replayable_scn:1733118915064689000, readable_scn:1733118914563953004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:16.434340] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.436066] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.436077] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.436092] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.436101] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.440629] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5F9-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.481320] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC50-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:16.541066] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.542490] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.542501] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.542517] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.542526] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.556638] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.614146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC30-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.617601] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC30-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.647418] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.649379] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B2-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.651573] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA85-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.654000] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA85-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.660142] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FB-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.754264] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B3-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.755652] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B3-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.759131] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE966-0-0] [lt=26] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:16.766155] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FC-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.856683] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118916853341, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118916853341, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:16.860406] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=34] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.861978] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.861988] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.862004] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.862013] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.869552] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FD-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.939146] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=33] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:16.940341] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=18] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:16.941502] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE966-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.955186] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC42-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:16.966828] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:16.968883] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.968894] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:16.968910] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:16.968919] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:16.973616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:16.981130] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FE-0-0] [lt=54] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.052803] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4A-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.073737] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.075609] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.075615] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.075631] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.075640] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B6-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.081480] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF5FF-0-0] [lt=24][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.101403] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2320160][T1001_BackupDat][T1001][YB42C0A800AF-000628430AAEE96A-0-0] [lt=17] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:17.167704] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:17.170186] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:17.170205] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC30-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:17.180360] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=13] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118916712394002, v:0}}], last_dump_ts_=1733118912096449, force_dump=false) [2024-12-02 13:55:17.180385] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:17.180476] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.181742] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.181753] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.181769] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.181778] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.185235] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF600-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.195828] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEA9F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.206366] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB8-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:17.236864] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4E-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.275158] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD7-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:17.286760] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.288399] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.288410] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.288426] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.288435] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.288662] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF601-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.312848] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEA9F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.352679] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:17.392097] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF602-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.393288] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.394571] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.394582] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.394598] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.394608] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2B9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.406914] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=16] start checking lost replicas [2024-12-02 13:55:17.481398] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC51-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.496069] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF603-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.499405] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.500502] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.500513] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.500527] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.500537] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.599796] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF604-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.605433] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.606386] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF604-0-0] [lt=53] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.607634] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.607644] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.607657] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.607666] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.622719] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC6-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:55:17.622772] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC31-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:17.625167] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC31-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:17.660766] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA86-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:17.663214] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA86-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:17.706873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF605-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.712652] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.733958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.733969] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.733986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.733995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.810849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF606-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.838916] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.840457] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.840468] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.840483] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.840493] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.867987] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACC-0-0] [lt=16] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:17.914554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF607-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.945446] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:17.946645] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:17.946655] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:17.946670] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:17.946676] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:17.956532] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC43-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.017871] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF608-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.032273] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC8-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.051621] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.052881] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.053172] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.053183] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.053199] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.053209] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2BF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.136687] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=19] failed to get result(ret=-4008) [2024-12-02 13:55:18.150849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF609-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.158083] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.159252] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.159263] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.159278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.159288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C0-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.172807] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:18.175295] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:18.175314] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC31-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:18.195902] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.212521] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB9-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:18.223632] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECB9-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:18.236965] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC4F-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.254548] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.264138] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.265394] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.265404] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.265419] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.265427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C1-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.312925] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA0-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.352731] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:18.352751] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=264, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:18.358425] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60B-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.370290] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.371544] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.371554] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.371566] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.371571] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.462178] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.476498] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.477703] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.477713] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.477728] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.477736] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C3-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.481476] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC52-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:18.566049] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.566260] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60D-0-0] [lt=46] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118913566166, force_dump=false) [2024-12-02 13:55:18.582529] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.584504] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.584511] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.584525] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.584533] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C4-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.630487] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC32-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:18.632858] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC32-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.668804] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.669834] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA87-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:18.672216] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA87-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:18.689239] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.690483] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.690490] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.690502] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.690512] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C5-0-0] [lt=10] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.772113] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF60F-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.795437] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:18.955559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:18.955570] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:18.955582] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:18.955588] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C6-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:18.957942] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC44-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.016907] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF610-0-0] [lt=43] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118913784625, force_dump=false) [2024-12-02 13:55:19.016944] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF610-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.052955] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4C-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.060371] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.062539] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.062550] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.062567] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.062576] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C7-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.120968] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF611-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.167387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.168640] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=13][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.168651] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.168668] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.168677] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.177914] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:19.180381] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:19.180399] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC32-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:19.195977] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.224559] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF612-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.228836] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:19.234643] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118919068906001, v:0}}], last_dump_ts_=1733118914133892, force_dump=false) [2024-12-02 13:55:19.234667] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBA-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:55:19.237062] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC50-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.273502] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.274967] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.274974] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.274989] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.274995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2C9-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.291137] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE998-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118919290940000, start_id=1733118919290940000, end_id=1733118919290940001) [2024-12-02 13:55:19.302397] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE998-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118919290940000, v:0}, tenant_id=1) [2024-12-02 13:55:19.312999] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA1-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.328124] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF613-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.352565] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118919349867, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118919349867, ora_rowscn_=1733118919231017000, content_changed=true) [2024-12-02 13:55:19.352831] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:19.379848] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.381088] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.381099] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.381115] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.381125] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.411460] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118919349867, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118918067229001, replayable_scn:1733118918067229001, readable_scn:1733118918067229000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:19.431665] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF614-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.481550] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC53-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.486067] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.487800] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.487807] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.487824] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.487833] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.535007] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF615-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.592713] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.601668] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.601678] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.601694] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.601703] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.623139] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC7-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:19.638068] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC33-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:19.638714] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF616-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.640476] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC33-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.678929] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA88-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:19.681352] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA88-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:19.706440] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.708242] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.708252] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.708269] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.708278] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.742278] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF617-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.813101] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.814277] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.814288] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.814303] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.814308] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.846196] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF618-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.858272] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118919855057, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118919855057, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:19.872822] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACD-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:19.919090] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:19.920807] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.920816] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:19.920832] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:19.920842] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2CF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:19.949849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF619-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:19.959319] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC45-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.025666] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.027070] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.027081] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.027097] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.027106] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.032614] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAC9-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.053038] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4D-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.053602] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.131899] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.133400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.133407] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.133418] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.133427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.156972] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61B-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.182997] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:20.185435] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:20.185454] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC33-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:20.196057] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA2-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.237160] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC51-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.238223] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.239453] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.239463] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.239478] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.239484] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.239646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBB-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:20.245168] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBB-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:20.260901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.313076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA2-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.344310] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.345722] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.345733] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.345750] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.345759] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.352864] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:20.364729] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.450499] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.451941] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.451952] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.451968] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.451977] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.468124] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.481627] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC54-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:20.556747] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.558997] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.559007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.559023] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.559033] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.571346] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF61F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.645674] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC34-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:20.648070] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC34-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.663794] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.665296] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.665304] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.665319] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.665328] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.675078] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF620-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.688051] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA89-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:20.690479] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA89-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.770073] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.771868] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.771878] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.771894] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.771906] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D7-0-0] [lt=11] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.778412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF621-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.876712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.878231] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.878238] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.878254] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.878263] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.882059] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF622-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.960070] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:20.960636] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC46-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:20.983067] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:20.984375] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.984386] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:20.984402] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:20.984410] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2D9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:20.985341] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF623-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:20.992528] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:55:21.053118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4E-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.088742] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF624-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.089363] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.090630] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.090640] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.090653] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.090663] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.132978] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.134385] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:21.134443] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:21.134455] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:21.134466] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:21.134470] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:21.134478] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:21.134485] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118926132998, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:21.188233] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=33] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:21.190804] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:21.190827] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC34-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:21.191502] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF625-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.195672] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.196133] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA3-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.196234] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.196245] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.196262] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.196271] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.212762] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.213901] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921212698, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:21.213925] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926213922, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:21.221600] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96E-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118921220846000, start_id=1733118921220846000, end_id=1733118921220846001) [2024-12-02 13:55:21.223143] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96E-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118921220846000, v:0}, tenant_id=1019) [2024-12-02 13:55:21.237263] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC52-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.262487] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBC-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:21.268397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBC-0-0] [lt=27] is primary tenant, no need check readable_scn [2024-12-02 13:55:21.295283] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF626-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.301295] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.302552] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.302563] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.302578] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.302587] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.307654] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:21.308790] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921307590, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:21.308815] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926308813, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:21.309932] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118921308695, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:21.309955] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926309953, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:21.313154] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA3-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.352779] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:21.354068] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:21.398571] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF627-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.407544] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.409071] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.409082] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.409098] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.409106] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DD-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.481700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC55-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.502846] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF628-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.514111] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.515609] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.515620] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.515636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.515645] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.596874] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=21] failed to get result(ret=-4008) [2024-12-02 13:55:21.606152] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF629-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.620676] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.621846] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.621853] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.621869] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.621879] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2DF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.623556] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAC8-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:21.653425] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC35-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:21.655826] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC35-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:21.697187] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8A-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:21.699588] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8A-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:21.709919] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.726751] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.728253] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.728264] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.728279] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.728288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.813448] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.833193] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.834522] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.834533] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.834548] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.834557] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.877484] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACE-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:21.916794] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.939571] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:21.941731] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:21.941741] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:21.941753] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:21.941761] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E2-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:21.961981] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC47-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.020234] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.032933] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACA-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.046622] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.047921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.047928] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.047940] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.047945] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E3-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.053198] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC4F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.123747] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.152889] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.154612] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.154621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.154636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.154643] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E4-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.193503] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:22.195987] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:22.196003] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC35-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:22.196208] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.227296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF62F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.237336] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC53-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.259699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.269530] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.269537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.269548] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.269553] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E5-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.289075] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118922071217001, v:0}}], last_dump_ts_=1733118917180358, force_dump=false) [2024-12-02 13:55:22.289098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:22.295886] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBD-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:22.313229] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.330730] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF630-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.352696] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118922349950, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118922349950, ora_rowscn_=1733118922292078000, content_changed=true) [2024-12-02 13:55:22.352912] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:22.352929] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=208, return_code_array=[0], broadcast_times=2) [2024-12-02 13:55:22.374519] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.376135] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.376145] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.376160] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.376169] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E6-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.413103] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118922349950, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118921070407001, replayable_scn:1733118921070407001, readable_scn:1733118921070407001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:22.434040] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF631-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.468488] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDD9-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:22.481233] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.481776] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC56-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:22.482757] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.482767] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.482782] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.482790] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.537478] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF632-0-0] [lt=24][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.587683] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.589265] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.589274] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.589289] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.589295] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E8-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.640854] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF633-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.660933] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC36-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:22.663297] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC36-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.694237] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.695775] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.695786] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.695802] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.695811] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2E9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.706336] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8B-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:22.708761] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8B-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.744198] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF634-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.800687] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.801959] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.801970] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.801986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.801995] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.847857] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF635-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.859929] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118922857298, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118922857298, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:22.906891] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:22.908380] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.908391] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:22.908406] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:22.908416] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:22.953028] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF636-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:22.963337] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC48-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:22.984308] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=20] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=360, Holding Memory(MB)=61, Tenant Remain Memory(MB)=497, Usable Remain Memory(MB)=298, Last Update Limit Timestamp=1733118922983459, Is Updated=true) [2024-12-02 13:55:22.984471] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733118922983459000, start_id=1733118922983459000, end_id=1733118922983459001) [2024-12-02 13:55:23.002281] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE998-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118922983459000, v:0}, tenant_id=1001) [2024-12-02 13:55:23.013392] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.014861] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.014872] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.014888] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.014897] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.053276] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC50-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.056874] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF637-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.119800] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.121119] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.121129] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.121145] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.121155] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2ED-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.160240] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF638-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.196285] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA5-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.198673] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:23.201137] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:23.201156] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC36-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:23.226062] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.227709] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.227720] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.227736] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.227746] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.237437] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC54-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.263653] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF639-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.301219] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBE-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:23.306471] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBE-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:23.313304] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.332657] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.333901] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.333911] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.333927] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.333936] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2EF-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.352966] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:23.367005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.438912] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.440942] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.440953] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.440969] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.440979] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F0-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.470595] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.481850] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC57-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.545902] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.547509] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.547520] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.547535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.547544] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.574033] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.574241] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63C-0-0] [lt=48] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118918566259, force_dump=false) [2024-12-02 13:55:23.623981] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAC9-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:23.632049] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=22] failed to get result(ret=-4008) [2024-12-02 13:55:23.652430] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.653667] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.653678] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.653693] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.653699] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.668385] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC37-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:23.670803] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC37-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:23.678258] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63D-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.715521] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8C-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:23.719045] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8C-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:23.758368] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.774539] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.774550] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.774565] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.774573] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.781606] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63E-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.797039] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2B-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:55:23.797063] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=15] start try disaster recovery [2024-12-02 13:55:23.797074] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:23.798307] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.802864] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.802892] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.802902] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.802956] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:23.802966] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.802978] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.803013] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803028] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803037] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.803045] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.804962] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:23.804980] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:23.806258] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.810682] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.810709] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810720] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.810772] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:23.810785] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.810797] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.810831] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810846] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810855] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.810862] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.812723] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:23.814023] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:23.815227] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=18] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.819597] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.819625] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819636] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.819689] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:23.819701] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.819713] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.819746] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=25] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819761] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819769] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.819776] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.821648] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.828328] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:23.828356] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828367] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.828420] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:23.828429] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.828442] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.828474] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828489] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828497] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.828505] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.830273] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:23.830292] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:23.831514] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.835796] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.835823] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.835834] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.835885] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:23.835897] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.835910] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.835943] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.835958] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.835967] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.835975] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.837853] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:23.839074] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:23.840246] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=19] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.844527] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:23.844555] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.844565] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.844616] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:23.844627] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.844640] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.844672] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=25] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.844687] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.844695] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.844703] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.846532] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:23.853094] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:23.853121] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.853132] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:23.853183] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=49] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:23.853193] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:23.853206] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:23.853239] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.853255] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.853263] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.853272] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:23.855113] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:23.855133] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=17] start check unit migrate finish [2024-12-02 13:55:23.855145] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:23.860923] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:23.869555] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2324797][T1019_BackupDat][T1019][YB42C0A800AF-0006284313CEE95A-0-0] [lt=17] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:23.871136] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=20] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:23.879560] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.880835] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F4-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.880842] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F4-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.880858] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.880868] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F4-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:23.881620] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2C-0-0] [lt=16] start to check alter locality finish [2024-12-02 13:55:23.882394] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEACF-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.882744] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2D-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.883881] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2D-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.885018] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF63F-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.886264] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2F-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.887381] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC2F-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.889642] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC31-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.890753] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC31-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.891952] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC31-0-0] [lt=16] start check shrink resource pool [2024-12-02 13:55:23.891973] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC32-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:23.893038] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC32-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.894160] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC32-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.894238] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC33-0-0] [lt=17] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:23.895304] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC33-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.896414] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC33-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.896491] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC34-0-0] [lt=17] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:23.897561] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC34-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.898665] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC34-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.898742] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC34-0-0] [lt=15] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:23.898757] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC34-0-0] [lt=14] start check primary zone [2024-12-02 13:55:23.898769] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC35-0-0] [lt=7] start check primary zone(tenant_id=1) [2024-12-02 13:55:23.899823] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC35-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.899900] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC36-0-0] [lt=14] start check primary zone(tenant_id=1002) [2024-12-02 13:55:23.900954] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC36-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.901028] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=14] start check primary zone(tenant_id=1020) [2024-12-02 13:55:23.902091] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:23.902169] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=14] start do unit balance [2024-12-02 13:55:23.903134] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC37-0-0] [lt=12] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118923903122}) [2024-12-02 13:55:23.903253] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320115][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDDA-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:23.904636] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=15] start distribute for server status change [2024-12-02 13:55:23.904687] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=15] finish distribute for server status change(ret=0) [2024-12-02 13:55:23.904696] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=9] finish do unit balance(ret=0) [2024-12-02 13:55:23.907042] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=5] start to do tenant group unit balance [2024-12-02 13:55:23.907082] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=15] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:23.907097] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=9] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:23.907117] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=15] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:23.907142] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:23.907162] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:23.907173] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:23.907184] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:23.907208] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:23.907248] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=39] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:23.907259] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:23.907273] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:23.907286] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=11] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:23.964687] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC49-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:23.985664] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:23.988624] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF640-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.991683] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:23.991693] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:23.991709] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:23.991717] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.033244] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACB-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.053354] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC51-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.093236] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF641-0-0] [lt=52] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118919016906, force_dump=false) [2024-12-02 13:55:24.093278] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF641-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.096516] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.097060] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.097067] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.097081] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.097087] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.196354] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA6-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.196965] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF642-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.201901] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.203782] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC37-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:24.206305] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC37-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:24.206323] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC37-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:24.225079] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.225089] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.225102] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F7-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.225108] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F7-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.237547] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC55-0-0] [lt=44] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.300748] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF643-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.311535] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBF-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:24.313386] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA6-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.317227] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBF-0-0] [lt=14] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118924073146001, v:0}}], last_dump_ts_=1733118919234641, force_dump=false) [2024-12-02 13:55:24.317250] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECBF-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:24.329937] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.331801] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.331810] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.331826] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.331835] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.352997] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:24.404149] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF644-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.436621] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.437900] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.437910] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.437926] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.437934] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2F9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.481930] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC58-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:24.507543] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF645-0-0] [lt=23][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.542698] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FA-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.544276] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FA-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.544285] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FA-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.544302] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.544310] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.611258] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF646-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.649123] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.650379] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.650390] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.650402] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FB-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.650412] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.675914] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC38-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:24.678312] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC38-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:24.714717] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF647-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.725794] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8D-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:24.728203] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8D-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:24.755251] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.756855] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.756866] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FC-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.756882] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.756891] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.818442] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF648-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.861764] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.862966] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FD-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.862977] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FD-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.862992] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.863001] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FD-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:24.932234] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF649-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.966104] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4A-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:24.967873] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:24.969399] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:24.969408] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FE-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:24.969424] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:24.969429] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.035679] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.053431] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC52-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.074306] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.085283] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.085293] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.085309] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.085317] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF2FF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.139188] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.190303] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF300-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.191611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF300-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.191622] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF300-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.191638] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF300-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.191647] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF300-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.196429] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.208956] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC38-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:25.211446] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC38-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:25.211465] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC38-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:25.237648] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC56-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.238751] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE998-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1733118925238200000, start_id=1733118925238200000, end_id=1733118925238200001) [2024-12-02 13:55:25.240314] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE998-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118925238200000, v:0}, tenant_id=1002) [2024-12-02 13:55:25.242846] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.296518] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF301-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.298148] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF301-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.298158] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF301-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.298174] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF301-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.298184] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF301-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.313459] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.322456] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC0-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:55:25.329273] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC0-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-02 13:55:25.346135] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.352849] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118924073146002, replayable_scn:1733118924073146002, readable_scn:1733118924073146001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118925349669, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118924073146002, replayable_scn:1733118924073146002, readable_scn:1733118924073146001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118925349669, ora_rowscn_=1733118925324395001, content_changed=true) [2024-12-02 13:55:25.353052] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:25.353068] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=197, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:25.403116] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF302-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.404949] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF302-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.404960] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF302-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.404976] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF302-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.404985] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF302-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.414715] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118925349669, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118924073146002, replayable_scn:1733118924073146002, readable_scn:1733118924073146001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:25.449544] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.482009] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC59-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.510023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF303-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.511597] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF303-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.511608] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF303-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.511624] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF303-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.511633] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF303-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.552925] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF64F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.616616] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF304-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.618175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF304-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.618186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF304-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.618201] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF304-0-0] [lt=14][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:25.618211] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF304-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.624426] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEACA-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:25.656380] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF650-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.683505] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC39-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:25.685904] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC39-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:25.723078] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF305-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.724296] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF305-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.724306] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF305-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.724321] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF305-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.724330] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF305-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.735007] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8E-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:25.737432] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:25.759748] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF651-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.829076] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF306-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.830745] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF306-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.830755] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF306-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.830771] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF306-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.830779] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF306-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.861571] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118925858976, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118925858976, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:25.863173] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF652-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.887196] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD0-0-0] [lt=18] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:25.935611] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF307-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:25.937010] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF307-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.937020] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF307-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:25.937036] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF307-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:25.937042] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF307-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:25.962824] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:25.966554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF653-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:25.967472] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4B-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:26.033564] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACC-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.041848] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF308-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.043335] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF308-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.043345] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF308-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.043360] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF308-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.043369] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF308-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.053484] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC53-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.069874] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF654-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.133063] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:26.134443] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:26.134501] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:26.134512] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:26.134522] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:26.134530] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:26.134540] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:26.134547] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118931133078, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:26.148280] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF309-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.149529] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF309-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.149535] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF309-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.149551] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF309-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.149560] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF309-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.173098] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF655-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.196507] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA8-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.213987] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:26.214141] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC39-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:26.215139] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926213922, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:26.215164] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931215161, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:26.216671] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC39-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:26.216691] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC39-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:26.237744] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC57-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.254448] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.255702] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.255712] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.255728] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.255736] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.278225] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF656-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.308871] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:26.310024] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926308813, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:26.310049] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931310047, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:26.311160] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118926309953, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:26.311182] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931311181, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:26.313545] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA8-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.334380] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC1-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:26.340686] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC1-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:26.353104] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:26.354420] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:26.360569] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.362138] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.362148] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.362163] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.362172] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.381466] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF657-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.466985] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.468222] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.468233] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.468248] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.468258] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30C-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.482083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:26.484972] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF658-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.573071] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.574720] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.574727] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30D-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.574743] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.574752] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.588157] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF659-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.679645] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.683134] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.683145] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.683160] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.683168] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30E-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.691044] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3A-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:26.691704] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.694473] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3A-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:26.744197] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8F-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:26.746619] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA8F-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:26.759218] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE967-0-0] [lt=25] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:26.787944] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.789501] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.789512] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.789527] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.789535] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF30F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.795009] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.894318] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF310-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:26.895825] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF310-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.895836] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF310-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:26.895852] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF310-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:26.895861] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF310-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.898745] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:26.945698] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE967-0-0] [lt=30] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:26.946876] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE967-0-0] [lt=15] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:26.948022] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE967-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:26.968864] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4C-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:27.000545] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF311-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.001837] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF311-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.001847] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF311-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.001862] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF311-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.001871] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF311-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.002298] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65D-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.005131] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65D-0-0] [lt=49] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.053560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC54-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.105500] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65E-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.106673] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF312-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.113103] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF312-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.113113] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF312-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.113128] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF312-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.113134] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF312-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.196585] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAA9-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.216197] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF65F-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.217945] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF313-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.219366] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF313-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.219376] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF313-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.219392] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF313-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.219400] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF313-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.219386] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3A-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:27.221885] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3A-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:27.221902] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3A-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:27.237844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC58-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.313620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAA9-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.319515] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF660-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.324157] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF314-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.325813] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF314-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.325823] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF314-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.325839] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF314-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.325848] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF314-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.346190] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC2-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118927076183002, v:0}}], last_dump_ts_=1733118922289073, force_dump=false) [2024-12-02 13:55:27.346211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC2-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:27.352596] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC2-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:55:27.353175] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:27.410815] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=19] start checking lost replicas [2024-12-02 13:55:27.422936] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF661-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.430531] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF315-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.450864] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF315-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.450874] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF315-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.450885] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF315-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.450890] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF315-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.482162] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5B-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.526307] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF662-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.555605] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF316-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.556856] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF316-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.556866] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF316-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.556881] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF316-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.556886] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF316-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.624867] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEACB-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:27.634075] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF663-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.637002] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF663-0-0] [lt=18] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.661743] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF317-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.663032] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF317-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.663048] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF317-0-0] [lt=14][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.663063] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF317-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.663072] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF317-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.699623] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3B-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:27.702032] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3B-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:27.737493] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF664-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.753398] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA90-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:27.755837] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA90-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:27.767712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF318-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.769037] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF318-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.769047] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF318-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.769063] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF318-0-0] [lt=14][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:27.769072] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF318-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.841179] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF665-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.873842] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF319-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.875645] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF319-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.875655] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF319-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.875671] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF319-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.875676] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF319-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:27.892019] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD1-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:27.944679] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF666-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.970249] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4D-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:27.980379] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:27.981629] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:27.981636] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31A-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:27.981651] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:27.981660] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.033883] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACD-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.048265] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF667-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.053634] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC55-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.086420] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.087981] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.087991] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.088006] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.088015] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.151951] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF668-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.192657] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.193940] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31C-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.193951] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.193967] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.193975] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.196664] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAA-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.224614] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3B-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:28.227155] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3B-0-0] [lt=48] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:28.227175] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3B-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:28.237941] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC59-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.255276] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF669-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.286209] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDDB-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:28.288848] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDDF-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:28.289054] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320102][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDE0-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:55:28.289625] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDE1-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:28.289911] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320102][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDE2-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:28.290671] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDE8-0-0] [lt=11] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:28.291429] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320107][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDE9-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:28.299257] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.300701] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31D-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.300712] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.300729] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.300734] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31D-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.313700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.353097] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118926075777002, replayable_scn:1733118926075777002, readable_scn:1733118926075777002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118928349891, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118926075777002, replayable_scn:1733118926075777002, readable_scn:1733118926075777002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118928349891, ora_rowscn_=1733118927348632000, content_changed=false) [2024-12-02 13:55:28.368805] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.373104] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC3-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:28.378318] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC3-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:28.405521] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.407064] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.407074] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.407090] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.407099] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.416330] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118928349891, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118926075777002, replayable_scn:1733118926075777002, readable_scn:1733118926075777002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:28.474023] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.482237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5C-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:28.511994] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31F-0-0] [lt=27] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.513333] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.513344] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.513360] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.513368] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF31F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.577640] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.577872] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66C-0-0] [lt=48] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118923574240, force_dump=false) [2024-12-02 13:55:28.618310] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF320-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.619808] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF320-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.619819] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF320-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.619835] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF320-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.619844] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF320-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.682895] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.707194] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3C-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:28.709569] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3C-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:28.724586] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF321-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.744241] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF321-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.744253] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF321-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.744269] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF321-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.744277] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF321-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.762615] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA91-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:28.765029] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA91-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:28.786172] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.849032] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF322-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.850288] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF322-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.850298] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF322-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.850315] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF322-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.850320] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF322-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.863147] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118928860365, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118928860365, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:28.889684] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF66F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.955073] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF323-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:28.956723] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF323-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:28.956733] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF323-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:28.956749] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF323-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:28.956758] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF323-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:28.971617] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4E-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:28.992937] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF670-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.053709] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC56-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.061542] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF324-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.062781] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF324-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.062792] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF324-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.062807] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF324-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.062816] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF324-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.108558] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF671-0-0] [lt=48] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118924093234, force_dump=false) [2024-12-02 13:55:29.108597] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF671-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.167602] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF325-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.174665] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF325-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.174672] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF325-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.174687] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF325-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.174693] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF325-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.196738] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAB-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.212143] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF672-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.229919] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:29.232510] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3C-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:29.232532] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3C-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:29.238045] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5A-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.279488] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF326-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.282308] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF326-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.282318] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF326-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.282333] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF326-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.282338] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF326-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.308277] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE999-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733118929307648000, start_id=1733118929307648000, end_id=1733118929307648001) [2024-12-02 13:55:29.311880] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE999-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118929307648000, v:0}, tenant_id=1) [2024-12-02 13:55:29.313781] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAB-0-0] [lt=24] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.316649] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF673-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.353285] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:29.353303] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=212, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:29.383854] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC4-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:29.387217] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF327-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.389922] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF327-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.389933] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF327-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.389948] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF327-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.389957] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF327-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.392047] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC4-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118929078197002, v:0}}], last_dump_ts_=1733118924317225, force_dump=false) [2024-12-02 13:55:29.392068] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC4-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:29.419936] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF674-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.482315] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5D-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.494669] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF328-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.495921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF328-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.495931] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF328-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.495946] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF328-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.495951] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF328-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.524712] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF675-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.600714] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF329-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.602198] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF329-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.602208] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF329-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.602224] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF329-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.602232] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF329-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.625284] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEACC-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:29.630116] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF676-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.707030] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.708278] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.708289] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.708306] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.708314] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.714696] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3D-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:29.717106] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3D-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:29.733737] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF677-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.771815] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA92-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:29.774241] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA92-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:29.813007] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.814526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.814537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.814553] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32B-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.814562] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.837085] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF678-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.896789] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD2-0-0] [lt=18] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:29.919304] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:29.920512] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.920523] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:29.920535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32C-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:29.920543] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:29.940300] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF679-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:29.973004] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC4F-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:30.025294] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.027276] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.027287] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.027304] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.027313] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.034206] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACE-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.043503] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.053783] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC57-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.132175] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.133393] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32E-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.133404] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.133420] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.133429] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.146756] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67B-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.196814] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.235231] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:30.237799] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3D-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:30.237819] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3D-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:30.238123] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.238143] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5B-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.239671] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32F-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.239678] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32F-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.239694] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.239703] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF32F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.249908] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67C-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.313858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAC-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.344412] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF330-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.345656] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF330-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.345667] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF330-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.345683] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF330-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.345692] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF330-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.353233] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67D-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.353415] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:30.397490] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC5-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:30.403279] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC5-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:30.450433] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF331-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.451779] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF331-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.451789] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF331-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.451805] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF331-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.451814] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF331-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.456842] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.482393] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5E-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:30.556543] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF332-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.558258] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF332-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.558268] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF332-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.558284] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF332-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.558293] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF332-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.560180] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF67F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.663085] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF333-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.663629] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF680-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.664907] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF333-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.664917] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF333-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.664934] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF333-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.664943] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF333-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.722374] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3E-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:30.724794] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3E-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:30.767379] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF681-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.769589] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF334-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.772937] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF334-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.772944] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF334-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.772959] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF334-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.772968] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF334-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.781097] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA93-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:30.783530] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA93-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:30.871772] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF682-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.877601] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF335-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.878836] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF335-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.878846] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF335-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.878862] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF335-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.878871] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF335-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:30.965416] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:30.974423] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC50-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:30.975318] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF683-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.979896] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=10] start do ObArchiveSchedulerService round(round=9) [2024-12-02 13:55:30.983386] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=20][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:55:30.983405] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=19][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1020 AND dest_no = 0 AND name = 'state', exec_tenant_id=1019) [2024-12-02 13:55:30.983474] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:30.983493] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=18][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:55:30.983505] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE954-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:30.983557] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF336-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:30.985212] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF336-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:30.985221] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF336-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:30.985236] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF336-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:30.985245] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF336-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.053858] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC58-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.078706] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF684-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.089986] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF337-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.091161] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF337-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.091173] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF337-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.091189] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF337-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.091198] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF337-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.133143] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:31.134556] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:31.134613] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:31.134629] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:31.134640] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:31.134649] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:31.134660] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:31.134668] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118936133160, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:31.182017] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF685-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.196069] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF338-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.196894] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAD-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.197334] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF338-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.197345] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF338-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.197361] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF338-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.197369] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF338-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.215223] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:31.216378] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931215161, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:31.216405] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936216403, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:31.228397] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96F-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118931228001000, start_id=1733118931228001000, end_id=1733118931228001001) [2024-12-02 13:55:31.229765] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE96F-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118931228001000, v:0}, tenant_id=1019) [2024-12-02 13:55:31.238229] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5C-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.240516] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3E-0-0] [lt=28] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:31.243076] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3E-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:31.243098] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3E-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:31.287239] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF686-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.302184] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF339-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.303840] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF339-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.303851] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF339-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.303867] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF339-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.303875] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF339-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.310111] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:31.311243] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931310047, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:31.311269] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936311267, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:31.312377] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118931311181, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:31.312401] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936312400, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:31.313934] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAD-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.353201] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118929078197002, replayable_scn:1733118929078197002, readable_scn:1733118929078197002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118931349994, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118929078197002, replayable_scn:1733118929078197002, readable_scn:1733118929078197002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118931349994, ora_rowscn_=1733118930400021000, content_changed=true) [2024-12-02 13:55:31.353490] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:31.354789] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:31.399971] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF687-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.408525] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC6-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:31.408669] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.410071] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.410081] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.410097] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.410106] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.413814] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC6-0-0] [lt=27] is primary tenant, no need check readable_scn [2024-12-02 13:55:31.417971] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118931349994, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118929078197002, replayable_scn:1733118929078197002, readable_scn:1733118929078197002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:31.468702] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDEB-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:31.482468] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC5F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.503356] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF688-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.515009] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.516289] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.516301] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33B-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.516317] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.516326] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.606999] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF689-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.621198] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.622745] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.622755] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.622769] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.622778] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.625691] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEACD-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:31.710501] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.727704] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33D-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.730156] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3F-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:31.732543] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC3F-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:31.733765] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33D-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.733775] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.733790] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.733798] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.790301] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA94-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:31.792703] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA94-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:31.814192] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.838580] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.840220] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.840230] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.840245] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.840253] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.864808] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118931862205, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118931862205, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:31.901568] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD3-0-0] [lt=18] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:31.918109] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.944981] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33F-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:31.946213] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:31.946222] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33F-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:31.946236] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:31.946245] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF33F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:31.975844] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC51-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:32.021584] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68D-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.034507] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEACF-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.051064] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF340-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.052607] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF340-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.052617] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF340-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.052631] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF340-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.052637] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF340-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.053934] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC59-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.125081] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.157626] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF341-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.158880] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF341-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.158890] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF341-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.158906] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF341-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.158914] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF341-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.196973] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAE-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.228628] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF68F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.238326] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5D-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.245807] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:32.248364] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3F-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:32.248384] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC3F-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:32.263832] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF342-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.265981] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF342-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.265992] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF342-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.266007] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF342-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.266014] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF342-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.314011] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAE-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.332280] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF690-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.353425] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:32.353443] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=235, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:32.370765] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF343-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.372097] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF343-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.372106] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF343-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.372120] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF343-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.372128] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF343-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.419755] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC7-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118932081861002, v:0}}], last_dump_ts_=1733118927346188, force_dump=false) [2024-12-02 13:55:32.419778] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC7-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:32.435823] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF691-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.441565] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC7-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:55:32.476941] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF344-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.478313] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF344-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.478322] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF344-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.478336] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF344-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.478341] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF344-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.482545] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC60-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:32.539794] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF692-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.583121] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF345-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.585043] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF345-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.585050] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF345-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.585064] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF345-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.585071] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF345-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.643154] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF693-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.689925] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF346-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.691170] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF346-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.691180] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF346-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.691195] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF346-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.691203] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF346-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.737906] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC40-0-0] [lt=10] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:32.740361] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC40-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:32.756753] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF694-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.796029] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF347-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.797269] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF347-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.797276] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF347-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.797290] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF347-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.797298] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF347-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.799584] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA95-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:32.802030] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA95-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:32.860516] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF695-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.901995] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF348-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:32.903751] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF348-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:32.903760] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF348-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:32.903775] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF348-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:32.903781] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF348-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:32.977231] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC52-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:32.980085] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF696-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.007759] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE999-0-0] [lt=18] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=358, Holding Memory(MB)=61, Tenant Remain Memory(MB)=495, Usable Remain Memory(MB)=297, Last Update Limit Timestamp=1733118933007148, Is Updated=true) [2024-12-02 13:55:33.007937] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE999-0-0] [lt=24] get number(ret=0, service_type_=0, range=1, base_id=1733118933007148000, start_id=1733118933007148000, end_id=1733118933007148001) [2024-12-02 13:55:33.008578] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF349-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.009789] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE999-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118933007148000, v:0}, tenant_id=1001) [2024-12-02 13:55:33.009810] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF349-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.009819] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF349-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.009834] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF349-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.009843] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF349-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.054005] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5A-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.083781] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF697-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.114724] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.116336] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.116343] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34A-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.116355] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34A-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.116363] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.187394] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF698-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.197054] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAAF-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.221101] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.235914] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34B-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.235924] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.235940] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.235946] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.238425] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5E-0-0] [lt=38] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.251053] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC40-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:33.253568] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC40-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:33.253586] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC40-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:33.290817] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF699-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.314078] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAAF-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.340749] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.342361] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.342372] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.342388] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.342393] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.353538] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:33.394434] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69A-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.446615] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC8-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:33.447249] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.448706] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.448716] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.448733] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.448742] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.452284] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC8-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:33.482620] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC61-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.497811] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69B-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.553745] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.555733] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34E-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.555743] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.555756] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.555761] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.601349] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.601580] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69C-0-0] [lt=50] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118928577871, force_dump=false) [2024-12-02 13:55:33.626137] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEACE-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:33.660628] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34F-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.662292] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.662302] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.662317] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34F-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.662326] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF34F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.704934] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.745722] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC41-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:33.748199] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC41-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:33.767037] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF350-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.768451] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF350-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.768461] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF350-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.768477] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF350-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.768486] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF350-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.808760] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA96-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:33.808895] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69E-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.812263] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA96-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:33.873159] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF351-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.874573] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF351-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.874583] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF351-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.874599] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF351-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.874607] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF351-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.906358] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD4-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:33.907822] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC37-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:55:33.907853] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=21] start try disaster recovery [2024-12-02 13:55:33.907866] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:33.909116] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.912708] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF69F-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.913771] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:33.913801] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.913813] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.913868] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=51] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:33.913877] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.913891] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.913926] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.913943] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.913953] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.913959] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.916053] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:33.916072] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:33.917357] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.921832] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:33.921862] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.921872] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.921928] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=52] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:33.921938] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.921952] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.921987] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.922003] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.922012] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.922021] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.924008] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:33.925309] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:33.926533] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.930925] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:33.930956] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.930969] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.931024] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=51] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:33.931038] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.931052] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.931090] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=31] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.931107] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.931116] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.931125] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.932868] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.939566] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:33.939597] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.939609] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.939661] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=49] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:33.939670] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.939684] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.939718] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.939734] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.939743] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.939752] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.941580] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:33.941599] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:33.942798] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.947044] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:33.947074] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.947085] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.947137] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=49] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:33.947146] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.947160] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.947194] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.947209] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.947218] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.947227] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.949149] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:33.950392] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:33.951568] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=22] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.955862] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=27] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:33.955892] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.955904] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.955957] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=49] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:33.955969] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.955983] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.956017] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.956034] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.956043] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.956052] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.957936] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:33.964714] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:33.964746] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.964757] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:33.964811] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=51] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:33.964821] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:33.964835] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:33.964870] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.964886] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.964897] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.964903] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:33.966820] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:33.966843] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=19] start check unit migrate finish [2024-12-02 13:55:33.966861] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:33.972535] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:33.978630] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC53-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:33.979598] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF352-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:33.980929] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF352-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:33.980940] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF352-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:33.980957] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF352-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:33.980966] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF352-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:33.982885] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=21] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:33.993117] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC38-0-0] [lt=18] start to check alter locality finish [2024-12-02 13:55:33.994272] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC39-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:33.995416] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC39-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:33.997771] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3B-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:33.998895] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3B-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.001180] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3D-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.002319] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3D-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.003528] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3D-0-0] [lt=18] start check shrink resource pool [2024-12-02 13:55:34.003551] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3E-0-0] [lt=18] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:34.004621] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3E-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.005762] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3E-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.005844] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3F-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:34.006910] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3F-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.008024] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC3F-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.008105] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC40-0-0] [lt=18] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:34.009168] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC40-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.010296] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC40-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.010376] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC40-0-0] [lt=17] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.010393] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC40-0-0] [lt=15] start check primary zone [2024-12-02 13:55:34.010407] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC41-0-0] [lt=8] start check primary zone(tenant_id=1) [2024-12-02 13:55:34.011462] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC41-0-0] [lt=5][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.011542] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC42-0-0] [lt=17] start check primary zone(tenant_id=1002) [2024-12-02 13:55:34.012603] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC42-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.012682] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=17] start check primary zone(tenant_id=1020) [2024-12-02 13:55:34.013760] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:34.013840] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=17] start do unit balance [2024-12-02 13:55:34.014813] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC43-0-0] [lt=13] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118934014800}) [2024-12-02 13:55:34.015008] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDEC-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:34.016347] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=17] start distribute for server status change [2024-12-02 13:55:34.016314] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A0-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.016404] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=19] finish distribute for server status change(ret=0) [2024-12-02 13:55:34.016415] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=11] finish do unit balance(ret=0) [2024-12-02 13:55:34.018797] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=9] start to do tenant group unit balance [2024-12-02 13:55:34.018840] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=19] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:34.018852] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=7] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:34.018869] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=10] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:34.018896] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=14] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:34.018919] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:34.018931] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:34.018943] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:34.018967] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:34.019006] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=38] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:34.019013] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=7] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:34.019029] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:34.019042] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=13] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:34.034808] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD0-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.054076] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5B-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.085877] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF353-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.087290] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF353-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.087301] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF353-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.087316] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF353-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.087325] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF353-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.119934] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A1-0-0] [lt=52] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118929108557, force_dump=false) [2024-12-02 13:55:34.119969] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A1-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.192252] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF354-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.193565] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF354-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.193577] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF354-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.193593] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF354-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.193602] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF354-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.197130] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.223257] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A2-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.238516] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC5F-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.256217] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC41-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:34.258747] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC41-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:34.258769] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC41-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:34.298539] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF355-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.299814] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF355-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.299825] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF355-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.299840] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF355-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.299849] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF355-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.314152] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.327067] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A3-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.353404] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118932081861003, replayable_scn:1733118932081861003, readable_scn:1733118932081861003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118934350712, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118932081861003, replayable_scn:1733118932081861003, readable_scn:1733118932081861003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118934350712, ora_rowscn_=1733118933449107000, content_changed=true) [2024-12-02 13:55:34.353655] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:34.404684] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF356-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.406473] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF356-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.406483] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF356-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.406499] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF356-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.406508] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF356-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.419618] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118934350712, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118932081861003, replayable_scn:1733118932081861003, readable_scn:1733118932081861003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:34.431482] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.457402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC9-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:34.462905] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC9-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118934083530003, v:0}}], last_dump_ts_=1733118929392045, force_dump=false) [2024-12-02 13:55:34.462927] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECC9-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:34.482699] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC62-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:34.511472] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF357-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.512733] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF357-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.512743] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF357-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.512760] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF357-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.512769] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF357-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.535116] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A5-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.617628] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF358-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.619818] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF358-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.619825] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF358-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.619841] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF358-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.619850] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF358-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.638556] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A6-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.724701] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF359-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.726817] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF359-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.726828] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF359-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.726844] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF359-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.726849] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF359-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.746873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A7-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.753575] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC42-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:34.756027] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC42-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:34.818999] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA97-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:34.821399] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA97-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:34.831631] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.850666] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.851008] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.851018] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.851035] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.851040] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35A-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.866498] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118934864121, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118934864121, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:34.954478] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6A9-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.956006] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:34.957984] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:34.957995] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:34.958011] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:34.958022] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:34.980030] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC54-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:35.054136] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.059576] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AA-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.062915] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.063798] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.063809] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.063824] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.063833] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.163459] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.168740] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.170248] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.170258] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.170275] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.170283] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.197207] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB1-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.238608] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC60-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.245674] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE999-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118935244749000, start_id=1733118935244749000, end_id=1733118935244749001) [2024-12-02 13:55:35.247548] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE999-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118935244749000, v:0}, tenant_id=1002) [2024-12-02 13:55:35.261337] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC42-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:35.263800] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC42-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:35.263821] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC42-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:35.267184] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.275162] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.275702] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.275709] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35E-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.275724] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.275730] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.314223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB1-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.353675] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:35.370471] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.380571] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.381905] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.381915] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.381930] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.381937] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF35F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.467957] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCA-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:55:35.473107] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCA-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:35.474033] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.482771] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC63-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.486786] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF360-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.488220] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF360-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.488230] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF360-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.488245] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF360-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.488253] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF360-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.576827] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6AF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.593042] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF361-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.600984] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF361-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.600994] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF361-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.601010] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF361-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.601019] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF361-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.626578] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEACF-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:35.680174] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B0-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.705847] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF362-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.707353] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF362-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.707362] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF362-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.707377] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF362-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.707385] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF362-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.761412] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC43-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:35.763890] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC43-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:35.783879] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B1-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.812191] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF363-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.813420] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF363-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.813429] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF363-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.813444] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF363-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.813452] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF363-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.828129] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA98-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:35.830563] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA98-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:35.888878] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.911234] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD5-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:35.918232] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF364-0-0] [lt=23] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:35.920513] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF364-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:35.920522] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF364-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:35.920538] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF364-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:35.920545] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF364-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:35.968260] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:35.981440] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC55-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:35.992538] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B3-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.025381] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF365-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.026645] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF365-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.026652] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF365-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.026664] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF365-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.026669] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF365-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.035141] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD1-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.054209] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.110756] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.131697] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF366-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.133225] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:36.133796] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF366-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.133807] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF366-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.133823] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF366-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.133832] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF366-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.134632] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:36.134689] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:36.134701] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:36.134712] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:36.134720] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:36.134731] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:36.134735] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118941133244, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:36.197283] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.214697] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.216468] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:36.217647] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936216403, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:36.217673] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941217671, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:36.238679] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF367-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.238700] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC61-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.239887] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF367-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.239897] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF367-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.239913] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF367-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.239922] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF367-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.266466] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC43-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:36.268987] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC43-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:36.269007] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC43-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:36.311329] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:36.312504] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936311267, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:36.312531] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941312529, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:36.313684] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118936312400, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:36.313709] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941313707, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:36.314296] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.318093] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B6-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.344815] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF368-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.346364] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF368-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.346375] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF368-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.346390] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF368-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.346399] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF368-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.353684] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:36.353702] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=223, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:36.355012] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:36.421663] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.451475] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF369-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.453191] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF369-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.453198] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF369-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.453215] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF369-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.453225] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF369-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.478885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCB-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:36.482846] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC64-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:36.485176] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCB-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:36.524940] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.558101] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.559526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.559537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.559552] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.559561] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.628496] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6B9-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.664501] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.666170] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.666177] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.666193] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.666199] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.731829] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BA-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.759308] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE968-0-0] [lt=28] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:36.769260] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC44-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:36.771131] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.772390] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.772401] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.772417] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.772422] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.772805] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC44-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:36.835334] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.837200] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA99-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:36.839557] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA99-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:36.877313] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.878845] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36D-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.878855] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.878870] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.878880] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.938680] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BC-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.952282] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE968-0-0] [lt=28] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:36.953517] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE968-0-0] [lt=17] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:36.954684] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE968-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:36.982846] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC56-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:36.983762] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:36.992478] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:36.992490] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:36.992502] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:36.992507] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36E-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.042091] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BD-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.045045] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BD-0-0] [lt=52] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.054267] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.069946] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF2-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:37.097442] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36F-0-0] [lt=13] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.098971] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.098982] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.098998] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.099007] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF36F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.145410] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BE-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.197362] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.203925] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF370-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.210018] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF370-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.210029] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF370-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.210045] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF370-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.210054] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF370-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.238789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC62-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.248749] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6BF-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.271686] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC44-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:37.274240] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC44-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:37.274259] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC44-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:37.314370] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.314864] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF371-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.316607] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF371-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.316618] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF371-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.316633] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF371-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.316642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF371-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.352081] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C0-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.353564] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118935246858001, replayable_scn:1733118935246858001, readable_scn:1733118935246858001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118937350427, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118935246858001, replayable_scn:1733118935246858001, readable_scn:1733118935246858001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118937350427, ora_rowscn_=1733118936481347000, content_changed=true) [2024-12-02 13:55:37.353776] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:37.414693] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=16] start checking lost replicas [2024-12-02 13:55:37.421266] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118937350427, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118935246858001, replayable_scn:1733118935246858001, readable_scn:1733118935246858001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:37.421489] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF372-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.422717] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF372-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.422728] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF372-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.422743] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF372-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.422751] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF372-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.455543] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C1-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.482920] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC65-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.490451] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCC-0-0] [lt=10] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118937086396003, v:0}}], last_dump_ts_=1733118932419753, force_dump=false) [2024-12-02 13:55:37.490472] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCC-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:55:37.495889] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCC-0-0] [lt=16] is primary tenant, no need check readable_scn [2024-12-02 13:55:37.527730] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF373-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.529134] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF373-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.529144] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF373-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.529159] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF373-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.529167] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF373-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.559600] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.627035] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAD0-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=19, ddl_arg=NULL) [2024-12-02 13:55:37.634146] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF374-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.635995] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF374-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.636005] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF374-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.636019] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF374-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.636025] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF374-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.662983] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.665912] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C3-0-0] [lt=52] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.740917] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF375-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.742461] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF375-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.742471] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF375-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.742485] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF375-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.742493] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF375-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.766343] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.778050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC45-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:37.780467] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC45-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:37.846109] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9A-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:37.847561] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF376-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.848447] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9A-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:37.849222] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF376-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.849229] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF376-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.849245] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF376-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.849254] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF376-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.868109] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118937865030, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118937865030, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:37.869675] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.916085] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD6-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:37.940734] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:37.954194] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF377-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:37.955464] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF377-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.955475] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF377-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:37.955490] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF377-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:37.955499] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF377-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:37.971271] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2319871][T1_BackupDataSr][T1][YB42C0A800AF-0006284308DEE967-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:37.973310] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:37.984256] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC57-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:38.035453] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD2-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.054346] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC5F-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.060445] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF378-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.061902] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF378-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.061913] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF378-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.061929] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF378-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.061939] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF378-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.076978] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C7-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.138042] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=22] failed to get result(ret=-4008) [2024-12-02 13:55:38.166876] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF379-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.168556] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF379-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.168567] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF379-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.168583] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF379-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.168588] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF379-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.180295] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.197440] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB4-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.238880] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC63-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.273681] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37A-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.274968] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.274979] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.274995] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.275004] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.276924] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC45-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:38.279475] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC45-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:38.279497] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC45-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:38.284031] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6C9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.314444] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.353955] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:38.380249] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.382489] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.382500] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.382516] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37B-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.382526] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.387504] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.482992] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC66-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:38.487606] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.488925] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.488936] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.488950] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.488958] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.491412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.500751] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCD-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:38.506308] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCD-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:38.593971] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37D-0-0] [lt=29] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.594765] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CC-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.595201] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37D-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.697483] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CD-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.697709] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CD-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118933601579, force_dump=false) [2024-12-02 13:55:38.700269] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.701481] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37E-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.785752] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC46-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:38.788188] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC46-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:38.801006] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.806509] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37F-0-0] [lt=26] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.807759] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF37F-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.855073] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9B-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:38.857420] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9B-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:38.905067] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6CF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.912622] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF380-0-0] [lt=26] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:38.922329] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF380-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:38.922340] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF380-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:38.922357] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF380-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:38.922366] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF380-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:38.985685] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC58-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:39.009412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.027270] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF381-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.028516] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF381-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.028526] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF381-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.028542] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF381-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.028551] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF381-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.054424] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC60-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.112793] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.133504] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF382-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.135022] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF382-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.135034] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF382-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.135049] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF382-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.135055] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF382-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.197515] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.216504] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D2-0-0] [lt=50] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118934119933, force_dump=false) [2024-12-02 13:55:39.216542] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D2-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.238973] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC64-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.239869] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF383-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.245832] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF383-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.245843] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF383-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.245860] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF383-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.245869] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF383-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.282151] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC46-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:39.284673] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC46-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:39.284695] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC46-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:39.314513] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB5-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.317830] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99A-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1733118939316807000, start_id=1733118939316807000, end_id=1733118939316807001) [2024-12-02 13:55:39.320420] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D3-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.329736] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99A-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118939316807000, v:0}, tenant_id=1) [2024-12-02 13:55:39.350780] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF384-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.353129] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF384-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.353140] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF384-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.353157] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF384-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.353166] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF384-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.353784] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:39.353800] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=228, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:39.429194] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D4-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.458091] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF385-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.459333] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF385-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.459344] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF385-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.459360] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF385-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.459369] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF385-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.483063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC67-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.532887] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.534460] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCE-0-0] [lt=10] is primary tenant, no need check readable_scn [2024-12-02 13:55:39.564275] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF386-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.604141] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF386-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.604153] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF386-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.604166] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF386-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.604175] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF386-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.604255] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCE-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118939088630004, v:0}}], last_dump_ts_=1733118934462903, force_dump=false) [2024-12-02 13:55:39.604277] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCE-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:39.627466] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAD1-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:39.672145] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.709155] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF387-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.710953] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF387-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.710964] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF387-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.710981] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF387-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.710990] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF387-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.775952] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.793576] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC47-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:39.796046] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC47-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:39.815886] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF388-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.817395] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF388-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.817405] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF388-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.817421] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF388-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.817430] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF388-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.864200] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9C-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:39.866643] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9C-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:39.879189] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D8-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.920996] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD7-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:39.922318] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF389-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:39.924440] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF389-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.924451] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF389-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:39.924463] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF389-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:39.924472] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF389-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:39.982491] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6D9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:39.987068] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC59-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:40.029285] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.030934] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.030945] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.030961] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.030970] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38A-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.035753] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD3-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.054503] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC61-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.086035] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DA-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.135811] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.137697] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.137708] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.137724] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.137729] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.189254] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.197597] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB6-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.239062] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC65-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.242464] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.243773] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.243784] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.243799] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.243805] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.287363] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC47-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:40.289839] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC47-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:40.289860] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC47-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:40.292762] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.314584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.348551] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.350355] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.350366] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.350382] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.350391] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.353645] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118938087055003, replayable_scn:1733118938087055003, readable_scn:1733118938087055003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118940350725, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118938087055003, replayable_scn:1733118938087055003, readable_scn:1733118938087055003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118940350725, ora_rowscn_=1733118939536583000, content_changed=true) [2024-12-02 13:55:40.353877] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:40.396042] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DD-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.422927] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118940350725, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118938087055003, replayable_scn:1733118938087055003, readable_scn:1733118938087055003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:40.455127] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.463856] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.463867] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.463883] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.463895] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38E-0-0] [lt=12] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.468964] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF4-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:40.483135] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC68-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:40.499666] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.554772] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97A-0-0] [lt=16] [RESTORE] restore scheduler quit [2024-12-02 13:55:40.555967] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97A-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_recover_table_job) [2024-12-02 13:55:40.557099] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97B-0-0] [lt=19] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_import_table_job) [2024-12-02 13:55:40.557147] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97C-0-0] [lt=20][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733118940557121, ref_timestamp=1733118880552551, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:55:40.557235] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97D-0-0] [lt=14] [RESTORE] restore scheduler start [2024-12-02 13:55:40.557246] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97E-0-0] [lt=10] [RESTORE] try process restore job [2024-12-02 13:55:40.558363] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97E-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-12-02 13:55:40.568637] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38F-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.570197] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.570207] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.570223] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.570231] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF38F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.615135] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6DF-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.617603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCF-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:40.623649] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECCF-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:40.674913] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF390-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.676580] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF390-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.676590] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF390-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:40.676606] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF390-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:40.676615] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF390-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.718405] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E0-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.781411] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF391-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.782939] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF391-0-0] [lt=1] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.801412] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC48-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:40.803876] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC48-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:40.822857] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E1-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.869694] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118940867278, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118940867278, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:40.873547] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9D-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:40.876012] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9D-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:40.887744] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF392-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.889237] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF392-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:40.931561] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:40.948990] INFO [RS] do_work (ob_tenant_snapshot_scheduler.cpp:123) [2324817][T1019_SnapSche][T1019][YB42C0A800AF-00062843140EE952-0-0] [lt=16] tenant status is not valid(user_tenant_id=1020, status_satisfied=false) [2024-12-02 13:55:40.970943] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:40.988458] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5A-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:40.993868] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:55:40.994097] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF393-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:40.995296] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF393-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.035272] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.054583] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC62-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.100135] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF394-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.102654] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF394-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.102664] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF394-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.102680] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF394-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.102689] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF394-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.123040] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE966-0-0] [lt=27] [RESTORE] restore scheduler quit [2024-12-02 13:55:41.124256] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE966-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1019, sql=select * from __all_recover_table_job) [2024-12-02 13:55:41.125394] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE967-0-0] [lt=18] get targets(ret=0, jobs=[], exec_tenant_id=1019, sql=select * from __all_import_table_job) [2024-12-02 13:55:41.125440] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE968-0-0] [lt=18][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733118941125414, ref_timestamp=1733118881119864, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:55:41.125529] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE969-0-0] [lt=14] [RESTORE] restore scheduler start [2024-12-02 13:55:41.125539] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=10] [RESTORE] try process restore job [2024-12-02 13:55:41.126821] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=1) [2024-12-02 13:55:41.128184] INFO [RS.RESTORE] check_all_ls_restore_to_consistent_scn_finish_ (ob_restore_scheduler.cpp:1402) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=15] check all ls restore to consistent_scn not finish, just wait(ret=0, ret="OB_SUCCESS", tenant_id=1020, tenant_restore_status={status:0}) [2024-12-02 13:55:41.128205] INFO [RS.RESTORE] process_restore_job (ob_restore_scheduler.cpp:216) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=19] [RESTORE] doing restore(ret=0, job={restore_key:{tenant_id:1020, job_id:11}, initiator_job_id:11, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:3, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:55:41.133315] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:41.134708] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:41.134764] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:41.134775] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:41.134785] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:41.134792] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:41.134802] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:41.134808] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118946133337, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:41.138510] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.197674] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB7-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.207577] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF395-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.208958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF395-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.208969] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF395-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.208986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF395-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.208991] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF395-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.217737] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:41.218937] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941217671, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:41.218966] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946218963, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:41.234854] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE970-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1733118941234400000, start_id=1733118941234400000, end_id=1733118941234400001) [2024-12-02 13:55:41.236422] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE970-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118941234400000, v:0}, tenant_id=1019) [2024-12-02 13:55:41.239156] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC66-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.242139] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.292514] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC48-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:41.295043] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC48-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:41.295063] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC48-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:41.312594] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:41.313776] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941312529, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:41.313802] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946313799, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:41.313893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF396-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.314654] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB7-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.314928] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118941313707, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:41.314953] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946314951, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:41.315939] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF396-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.315950] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF396-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.315965] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF396-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.315974] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF396-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.345340] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E6-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.353871] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:41.355125] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:41.420747] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF397-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.422625] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF397-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.422636] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF397-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.422648] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF397-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.422656] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF397-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.448909] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.483207] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC69-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.527503] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF398-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.529086] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF398-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.529097] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF398-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.529113] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF398-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.529121] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF398-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.552077] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E8-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.598307] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=20] failed to get result(ret=-4008) [2024-12-02 13:55:41.627867] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAD2-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:41.634046] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF399-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.635365] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF399-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.635375] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF399-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.635390] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF399-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.635399] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF399-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.642290] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD0-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:41.649102] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD0-0-0] [lt=26] is primary tenant, no need check readable_scn [2024-12-02 13:55:41.655410] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6E9-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.740221] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.741831] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.741842] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.741859] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39A-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.741869] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39A-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.759189] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6EA-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.809125] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC49-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:41.811572] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC49-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:41.846650] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.848230] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.848240] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.848256] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.848265] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.862525] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6EB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.882604] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9E-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:41.885005] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9E-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:41.925835] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD8-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:41.952991] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39C-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:41.954317] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39C-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.954328] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:41.954343] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:41.954352] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:41.965739] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6EC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:41.989880] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5B-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:42.036026] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD4-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.054661] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC63-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.059227] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.062362] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39D-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.062374] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39D-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.062390] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.062396] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39D-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.068903] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6ED-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.167225] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39E-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.169200] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.169211] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.169227] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.169236] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.172089] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6EE-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.197756] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB8-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.239246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC67-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.274164] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.275307] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6EF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.275731] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.275742] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.275759] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39F-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.275768] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF39F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.297736] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC49-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:42.300294] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC49-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:42.300316] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC49-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:42.314730] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB8-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.353949] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:42.353965] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=208, return_code_array=[0], broadcast_times=2) [2024-12-02 13:55:42.378293] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F0-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.380721] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.381613] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A0-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.381623] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.381640] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.381645] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A0-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.481691] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.483280] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6A-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:42.486623] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A1-0-0] [lt=13] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.506307] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.506317] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.506333] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.506342] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A1-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.605524] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F2-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.611350] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.612582] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A2-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.612589] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A2-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.612603] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A2-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.612612] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A2-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.654250] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD1-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118942591712000, v:0}}], last_dump_ts_=1733118937490450, force_dump=false) [2024-12-02 13:55:42.654271] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD1-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:55:42.660531] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD1-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:42.708913] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F3-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.717653] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.718840] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.718851] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.718866] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.718875] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.812205] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F4-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.816905] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4A-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:42.819352] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4A-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:42.823974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.825327] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.825338] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.825354] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A4-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.825363] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.891660] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9F-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:42.894062] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEA9F-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:42.916584] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F5-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.930303] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:42.931580] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:42.931591] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:42.931607] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:42.931616] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:42.991278] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5C-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:43.015097] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99A-0-0] [lt=19] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=358, Holding Memory(MB)=61, Tenant Remain Memory(MB)=495, Usable Remain Memory(MB)=297, Last Update Limit Timestamp=1733118943014510, Is Updated=true) [2024-12-02 13:55:43.015277] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99A-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1733118943014510000, start_id=1733118943014510000, end_id=1733118943014510001) [2024-12-02 13:55:43.020008] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.026209] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99A-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118943014510000, v:0}, tenant_id=1001) [2024-12-02 13:55:43.036640] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.037830] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.037840] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.037856] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.037861] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.054734] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC64-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.124671] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F7-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.142925] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.144537] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.144547] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.144563] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.144572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A7-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.197834] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAB9-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.228018] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F8-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.239331] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC68-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.249500] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.250885] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.250896] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.250912] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.250921] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A8-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.287482] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320101][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF6-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:43.287713] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF7-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:55:43.291851] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF8-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:43.292673] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDF9-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:43.292874] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEDFA-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:55:43.293766] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE00-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:43.294394] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE01-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:43.294718] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE02-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:55:43.295301] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE03-0-0] [lt=4] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:55:43.295554] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE04-0-0] [lt=7] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:43.296437] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:55:43.303083] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4A-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:43.305582] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4A-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:43.305604] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4A-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:43.314806] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAB9-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.331994] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6F9-0-0] [lt=16][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:43.353932] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118941591038000, replayable_scn:1733118941591038000, readable_scn:1733118941591038000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118943350308, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118941591038000, replayable_scn:1733118941591038000, readable_scn:1733118941591038000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118943350308, ora_rowscn_=1733118942656924000, content_changed=true) [2024-12-02 13:55:43.354157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:43.355888] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.357223] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.357235] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.357251] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.357260] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3A9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.424595] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118943350308, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118941591038000, replayable_scn:1733118941591038000, readable_scn:1733118941591038000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:43.462307] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.463591] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.463602] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.463618] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.463628] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.483355] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.568765] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AB-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.570375] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.570383] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AB-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.570399] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.570408] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AB-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.628271] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAD3-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:43.633304] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=22] failed to get result(ret=-4008) [2024-12-02 13:55:43.643042] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6FC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.675405] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.676794] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AC-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.676802] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AC-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.676815] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AC-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.676823] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AC-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.680603] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD2-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:43.685755] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD2-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:43.746390] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6FD-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.746666] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6FD-0-0] [lt=52] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118938697708, force_dump=false) [2024-12-02 13:55:43.781817] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.783107] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.783117] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AD-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.783132] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AD-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.783138] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AD-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.824679] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4B-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:43.827096] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4B-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:43.850234] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6FE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.871395] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118943868160, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118943868160, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:43.888159] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.889627] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.889637] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.889651] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AE-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.889659] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AE-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:43.900688] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA0-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:43.901842] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:368) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA0-0-0] [lt=21] log restore source not exists or the log restore source type is not service(item={tenant_id:1020, id:1, until_scn:{val:1732874528445031000, v:0}, type:2, value:"file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"}) [2024-12-02 13:55:43.904220] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA0-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:43.930637] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAD9-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:43.953603] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF6FF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.992682] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5D-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:43.994581] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AF-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:43.995833] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:43.995843] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:43.995863] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AF-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:43.995871] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3AF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.019609] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC43-0-0] [lt=24] root balance idle(idle_time=10000000) [2024-12-02 13:55:44.019635] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=18] start try disaster recovery [2024-12-02 13:55:44.019648] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:44.020880] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.025391] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:44.025418] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.025425] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.025489] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=60] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:44.025499] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.025515] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=14] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.025551] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=29] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.025567] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.025578] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.025583] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.027490] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:44.027506] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:44.028783] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.033200] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:44.033224] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.033235] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.033288] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=50] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:44.033298] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.033311] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.033346] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.033362] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.033371] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.033380] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.035407] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:44.036368] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD5-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.036680] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:44.037881] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.042205] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:44.042232] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.042243] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.042296] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=51] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:44.042309] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.042322] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.042358] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.042374] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.042383] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.042392] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.044199] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.050791] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:44.050819] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.050830] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.050883] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:44.050893] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.050907] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.050941] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.050957] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.050966] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.050971] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.052995] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:44.053008] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:44.054212] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.054813] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC65-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.057362] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF700-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.058441] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:44.058466] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.058477] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.058531] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=49] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:44.058545] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.058559] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.058593] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.058610] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.058620] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.058628] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.060711] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:44.061912] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:44.063087] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.067446] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:44.067471] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.067482] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.067536] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:44.067545] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.067559] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.067593] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.067610] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.067620] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.067626] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.069654] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:44.076282] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:44.076307] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.076318] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:44.076371] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:44.076384] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:44.076397] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:44.076432] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.076447] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.076457] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.076466] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:44.078385] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:44.078404] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=15] start check unit migrate finish [2024-12-02 13:55:44.078420] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:44.084085] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:44.094309] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:44.101011] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B0-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.102615] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.102625] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.102640] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.102649] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B0-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.104391] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC44-0-0] [lt=18] start to check alter locality finish [2024-12-02 13:55:44.105531] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC45-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.106650] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC45-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.108952] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC47-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.110075] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC47-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.112310] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC49-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.113412] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC49-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.114616] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC49-0-0] [lt=16] start check shrink resource pool [2024-12-02 13:55:44.114636] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4A-0-0] [lt=14] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:44.115694] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4A-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.116799] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4A-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.116875] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4B-0-0] [lt=16] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:44.117940] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4B-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.119064] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4B-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.119138] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4C-0-0] [lt=14] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:44.120185] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4C-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.121279] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4C-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.121354] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4C-0-0] [lt=15] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.121369] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4C-0-0] [lt=14] start check primary zone [2024-12-02 13:55:44.121382] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4D-0-0] [lt=9] start check primary zone(tenant_id=1) [2024-12-02 13:55:44.122432] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4D-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.122507] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4E-0-0] [lt=14] start check primary zone(tenant_id=1002) [2024-12-02 13:55:44.123549] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4E-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.123625] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=14] start check primary zone(tenant_id=1020) [2024-12-02 13:55:44.124669] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:44.124746] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=16] start do unit balance [2024-12-02 13:55:44.127181] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=12] start distribute for server status change [2024-12-02 13:55:44.127232] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=16] finish distribute for server status change(ret=0) [2024-12-02 13:55:44.127243] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:55:44.129579] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=4] start to do tenant group unit balance [2024-12-02 13:55:44.129618] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=16] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:44.129634] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:44.129653] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=14] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:44.129678] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:44.129700] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:44.129711] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:44.129722] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=9] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:44.129746] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=16] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:44.129786] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=40] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:44.129794] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=8] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:44.129805] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=8] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:44.129820] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=13] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:44.161136] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF701-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.197908] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.198424] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=4] need remove task(remove_task_ids=[]) [2024-12-02 13:55:44.198439] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=15] remove all timeout ddl task succeed [2024-12-02 13:55:44.207663] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.208948] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B1-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.208958] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.208973] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.208982] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.239423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC69-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.264876] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF702-0-0] [lt=55] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118939216502, force_dump=false) [2024-12-02 13:55:44.264911] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF702-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.308223] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4B-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:44.310760] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4B-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:44.310781] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4B-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:44.313918] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.314878] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABA-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.315570] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.315580] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.315596] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.315605] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.354184] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:44.368267] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF703-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.400039] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2516) [2320738][DDLScanTask][T0][Y0-0000000000000000-0-0] [lt=19] start processing ddl recovery(ddl_event_info={addr_:"192.168.0.175:2882", event_ts_:1733118944400038, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1002]) [2024-12-02 13:55:44.420555] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.421839] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.421850] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.421866] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.421876] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B3-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.472333] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF704-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.483433] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6C-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:44.526748] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.528215] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B4-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.528222] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B4-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.528238] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.528247] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.575608] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF705-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.633297] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.634528] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.634539] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.634556] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.634565] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.679935] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF706-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.690902] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD3-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:44.696677] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD3-0-0] [lt=9] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118944593498000, v:0}}], last_dump_ts_=1733118939604253, force_dump=false) [2024-12-02 13:55:44.696696] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD3-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:44.739509] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B6-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.740762] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.740772] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.740788] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.740794] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.784074] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF707-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.832468] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4C-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:44.834957] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4C-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:44.845729] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.852383] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B7-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.852395] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.852411] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.852420] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B7-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.887582] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF708-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.910866] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA1-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:44.913221] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA1-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:44.957242] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:44.958462] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.958473] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:44.958489] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B8-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:44.958499] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:44.993398] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF709-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:44.994062] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5E-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:45.054890] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC66-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.063394] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.065333] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.065344] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.065359] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.065368] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3B9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.076429] INFO [RS] inner_dump_statistic_ (ob_disaster_recovery_task_mgr.cpp:1362) [2319637][DRTaskMgr][T0][Y0-0000000000000000-0-0] [lt=15] [DRTASK_NOTICE] disaster recovery task manager statistics(waiting_high_priority_task_cnt=0, executing_high_priority_task_cnt=0, waiting_low_priority_task_cnt=0, executing_low_priority_task_cnt=0) [2024-12-02 13:55:45.096848] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.170293] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.172541] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.172552] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BA-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.172567] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.172576] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.197988] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABB-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.200427] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70B-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.239512] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6A-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.253694] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99A-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733118945253397000, start_id=1733118945253397000, end_id=1733118945253397001) [2024-12-02 13:55:45.255692] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99A-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118945253397000, v:0}, tenant_id=1002) [2024-12-02 13:55:45.277362] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.278676] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.278686] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.278703] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.278712] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.304120] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.313385] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:45.314954] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABB-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.315898] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4C-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:45.315921] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4C-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:45.354205] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:45.383561] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.387041] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.387052] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BC-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.387068] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BC-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.387078] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BC-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.407458] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.483511] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6D-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.491957] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.493145] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.493156] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.493172] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.493181] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.511048] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.598145] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.600550] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.600557] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BE-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.600574] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.600580] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BE-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.614414] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF70F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.628704] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAD4-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:45.702021] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD4-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:55:45.705570] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.707273] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.707282] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.707299] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.707307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3BF-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.707364] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD4-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:55:45.717984] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF710-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.812396] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.813896] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C0-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.813908] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.813924] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.813934] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.821270] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF711-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.840167] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4D-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:45.842591] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4D-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:45.918617] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:45.919678] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA2-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:45.919883] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.919894] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:45.919910] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:45.919918] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:45.922003] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA2-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:45.925362] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF712-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:45.935422] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADA-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:45.973635] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:45.995473] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC5F-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:46.024697] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.026262] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.026273] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.026288] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.026297] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.028689] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF713-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.036664] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD6-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.054966] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC67-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.131086] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.132190] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF714-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.132360] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.132367] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C3-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.132382] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.132387] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C3-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.133402] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:46.134710] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:46.134763] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:46.134770] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:46.134778] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:46.134789] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:46.134799] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:46.134806] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118951133420, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:46.198068] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABC-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.219025] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:46.220193] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946218963, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:46.220217] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951220215, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:46.235616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF715-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.237258] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.238975] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.238986] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.238997] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C4-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.239003] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C4-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.239601] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6B-0-0] [lt=23] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.313862] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:46.315009] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABC-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.315013] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946313799, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:46.315033] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951315031, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:46.316161] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118946314951, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:46.316183] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951316181, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:46.318590] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4D-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:46.321099] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4D-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:46.321120] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4D-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:46.343803] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.345160] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.345171] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.345187] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.345192] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C5-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.352704] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF716-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.353934] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118944593498000, replayable_scn:1733118944593498000, readable_scn:1733118944593498000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118946350971, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118944593498000, replayable_scn:1733118944593498000, readable_scn:1733118944593498000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118946350971, ora_rowscn_=1733118945703900001, content_changed=true) [2024-12-02 13:55:46.354173] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:46.354189] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=234, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:46.355444] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:46.426194] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=19] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118946350971, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118944593498000, replayable_scn:1733118944593498000, readable_scn:1733118944593498000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:46.450051] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.451591] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.451601] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.451616] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C6-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.451623] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C6-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.456180] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF717-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.483559] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6E-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:46.556408] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.557656] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C7-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.557663] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C7-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.557678] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.557683] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C7-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.559791] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF718-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.662522] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C8-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.663218] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF719-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.665442] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C8-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.665453] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.665477] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C8-0-0] [lt=22][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.665483] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C8-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.712595] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD5-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:46.731973] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD5-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:55:46.759399] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE969-0-0] [lt=29] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:46.770311] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.771730] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.771741] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.771756] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.771766] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3C9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.782701] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.847758] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4E-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:46.851281] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4E-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:46.872893] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118946869676, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118946869676, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:46.876364] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.877636] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.877647] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.877662] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.877671] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.886025] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.928384] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA3-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:46.930688] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA3-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:46.958904] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE969-0-0] [lt=30] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:46.960119] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE969-0-0] [lt=17] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:46.961286] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE969-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.982383] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:46.984032] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.984042] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:46.984058] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CB-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:46.984068] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:46.993571] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:46.996878] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC60-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:47.055037] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC68-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.088801] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.090287] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.090298] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CC-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.090313] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.090321] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.097010] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.100209] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71D-0-0] [lt=51] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.102670] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2320160][T1001_BackupDat][T1001][YB42C0A800AF-000628430AAEE96B-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:47.111983] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:236) [2320159][T1001_BACKUP_SC][T1001][YB42C0A800AF-000628430A9EE94C-0-0] [lt=15] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-12-02 13:55:47.195056] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.197146] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.197156] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.197172] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.197180] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.198145] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABD-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.200584] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.239696] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6C-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.302037] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.303396] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.303407] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.303422] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.303430] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CE-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.315082] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABD-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.323769] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4E-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:47.324277] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF71F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.326287] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4E-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:47.326305] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4E-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:47.354230] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:47.408178] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.409428] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.409439] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CF-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.409455] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.409464] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3CF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.418429] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2024-12-02 13:55:47.427751] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF720-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.483633] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC6F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.514157] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D0-0-0] [lt=24] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.515673] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.515684] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.515699] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.515708] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.530992] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF721-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.620431] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.621678] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.621688] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.621703] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D1-0-0] [lt=13][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:55:47.621712] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.629139] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAD5-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:47.634815] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF722-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.726494] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.728011] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.728018] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D2-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.728033] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.728042] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.737185] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD6-0-0] [lt=18] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118947596776001, v:0}}], last_dump_ts_=1733118942654248, force_dump=false) [2024-12-02 13:55:47.737211] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD6-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:47.738153] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF723-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.741385] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF723-0-0] [lt=51] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.743853] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD6-0-0] [lt=10] is primary tenant, no need check readable_scn [2024-12-02 13:55:47.832811] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.834637] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D3-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.834648] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.834661] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D3-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.834669] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.841860] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF724-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.856535] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4F-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:47.858972] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC4F-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:47.937141] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA4-0-0] [lt=13] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:47.939223] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:47.939478] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA4-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:47.940314] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADB-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:47.940640] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.940650] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:47.940666] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:47.940675] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:47.945668] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF725-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:47.998298] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC61-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:48.036992] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD7-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.045412] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.047138] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.047149] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.047165] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.047174] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.049062] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF726-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.055114] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC69-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.151854] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.152608] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF727-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.153354] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.153364] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.153380] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.153389] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.177793] INFO [RS] process (ob_root_inspection.cpp:494) [2319627][RSInspectTask][T0][Y0-0006284310BEEF8F-0-0] [lt=4] purge recyclebin task begin [2024-12-02 13:55:48.177809] INFO [RS] process (ob_root_inspection.cpp:512) [2319627][RSInspectTask][T0][Y0-0006284310BEEF8F-0-0] [lt=12] submit purge recyclebin task success(delay=60000000) [2024-12-02 13:55:48.177814] INFO [RS] process (ob_root_inspection.cpp:514) [2319627][RSInspectTask][T0][Y0-0006284310BEEF8F-0-0] [lt=5] purge recyclebin task end(delay=60000000) [2024-12-02 13:55:48.198223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.239786] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6D-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.256004] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF728-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.258209] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.259821] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.259832] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.259847] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.259852] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D7-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.315158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.328956] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:48.331475] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4F-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:48.331494] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC4F-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:48.354276] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:48.359927] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF729-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.364633] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.365225] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.365235] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.365251] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.365260] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.463556] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72A-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.470092] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.483705] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC70-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:48.483924] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D9-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.483934] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.483949] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.483958] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3D9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.582763] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.588699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.590264] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.590275] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.590290] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DA-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.590299] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.685442] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72C-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.695124] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DB-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.696906] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.696916] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.696931] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.696940] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.750930] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD7-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:48.756342] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD7-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:48.789125] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72D-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.789376] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72D-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118943746665, force_dump=false) [2024-12-02 13:55:48.802169] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DC-0-0] [lt=21] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.803423] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DC-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.803434] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DC-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.803451] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.803457] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DC-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.864348] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC50-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:48.866770] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC50-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:48.892568] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.908455] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:48.909655] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.909666] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:48.909682] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:48.909691] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:48.946040] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA5-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:48.948394] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA5-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:48.998556] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF72F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:48.999676] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC62-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:49.014543] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.016899] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.016910] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.016927] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.016936] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.055184] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.116151] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF730-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.121962] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.123251] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.123262] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.123278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.123283] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3DF-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.198297] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEABF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.220104] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF731-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.228255] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.229590] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.229600] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.229616] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.229625] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.239887] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6E-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.315229] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEABF-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.323871] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF732-0-0] [lt=52] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118944264875, force_dump=false) [2024-12-02 13:55:49.323910] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF732-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.334248] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC50-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:49.334581] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.335469] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99B-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1733118949335006000, start_id=1733118949335006000, end_id=1733118949335006001) [2024-12-02 13:55:49.336044] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.336054] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.336071] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.336080] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.336847] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC50-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:49.336867] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC50-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:49.337247] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99B-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118949335006000, v:0}, tenant_id=1) [2024-12-02 13:55:49.354147] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118947596776001, replayable_scn:1733118947596776001, readable_scn:1733118947596776001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118949350805, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118947596776001, replayable_scn:1733118947596776001, readable_scn:1733118947596776001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118949350805, ora_rowscn_=1733118948753057001, content_changed=true) [2024-12-02 13:55:49.354387] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:49.427840] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=16] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118949350805, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118947596776001, replayable_scn:1733118947596776001, readable_scn:1733118947596776001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:49.428332] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF733-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.441013] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.442664] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.442674] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.442689] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.442698] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.469146] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE08-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:49.483778] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC71-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.531880] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF734-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.547694] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.549099] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.549110] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.549126] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.549135] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.629589] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAD6-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:49.635167] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF735-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.654142] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.655621] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.655632] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.655648] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.655656] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E4-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.746386] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF736-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.760583] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.761798] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD8-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:49.762341] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.762351] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.762367] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.762376] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.767053] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD8-0-0] [lt=19] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118949599027001, v:0}}], last_dump_ts_=1733118944696676, force_dump=false) [2024-12-02 13:55:49.767076] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD8-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:49.849872] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF737-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.867370] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:49.868596] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.868607] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:49.868623] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:49.868632] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:49.871956] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC51-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:49.874381] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC51-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:49.874613] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=8] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118949871466, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118949871466, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:49.945177] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADC-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:49.953381] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF738-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:49.955044] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA6-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:49.957428] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA6-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:49.973557] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.001033] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC63-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:50.017559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E7-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.017566] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E7-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.017578] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E7-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.017583] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E7-0-0] [lt=3] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.037296] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD8-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.055257] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6B-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.056735] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF739-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.122564] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E8-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.123876] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E8-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.123886] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.123902] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.123911] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.159992] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.198376] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC0-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.228903] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.230603] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.230614] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.230630] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.230639] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3E9-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.239995] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC6F-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.263565] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.315305] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC0-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.335577] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.336858] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.336869] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.336885] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.336893] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EA-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.339544] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC51-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:50.342062] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC51-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:50.342080] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC51-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:50.354432] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:50.354449] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=258, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:50.366858] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.441793] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.443115] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.443126] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.443142] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.443151] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.470161] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.483855] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC72-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:50.548013] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.549252] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.549262] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EC-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.549278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.549288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EC-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.573422] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73E-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.654248] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3ED-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.655782] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3ED-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.655793] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3ED-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.655809] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3ED-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.655818] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3ED-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.678614] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF73F-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.760734] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.762329] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.762339] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.762355] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.762364] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.772144] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD9-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:55:50.777824] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECD9-0-0] [lt=39] is primary tenant, no need check readable_scn [2024-12-02 13:55:50.781857] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF740-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.867253] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.879659] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC52-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:50.882093] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC52-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:50.885450] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF741-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.890383] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.890391] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EF-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.890407] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.890416] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3EF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:50.964005] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA7-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:50.966360] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA7-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:50.976266] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:50.992631] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF742-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.995293] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:50.996217] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:50.996228] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:50.996243] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:50.996252] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.002378] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC64-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:51.055329] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6C-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.096058] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF743-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.101201] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.101778] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.101788] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.101804] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.101814] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F1-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.133487] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:51.134876] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:51.134933] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:51.134944] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:51.134955] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:51.134963] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:51.134974] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:51.134981] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118956133504, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:51.198455] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC1-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.199492] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF744-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.206747] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.208382] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.208392] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.208408] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.208417] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.220283] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:51.221464] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951220215, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:51.221501] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=35] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956221499, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:51.240099] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC70-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.241589] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE971-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1733118951240723000, start_id=1733118951240723000, end_id=1733118951240723001) [2024-12-02 13:55:51.243173] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE971-0-0] [lt=22] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118951240723000, v:0}, tenant_id=1019) [2024-12-02 13:55:51.303074] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF745-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.313400] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.314607] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.314617] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.314633] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.314642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.315096] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:51.315380] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC1-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.316262] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951315031, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:51.316289] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956316287, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:51.317414] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118951316181, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:51.317436] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956317434, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:51.344730] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC52-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:51.347257] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC52-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:51.347277] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC52-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:51.354444] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:51.355701] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:51.406432] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF746-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.419542] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.421256] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.421263] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F4-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.421278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.421287] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.483929] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC73-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.516949] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF747-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.526216] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.527400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.527411] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.527426] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.527435] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.620292] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF748-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.630022] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAD7-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:55:51.632398] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.649247] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.649257] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.649272] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.649282] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.723803] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF749-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.754357] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.755816] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F7-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.755822] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F7-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.755838] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.755847] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.783159] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDA-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:55:51.788527] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDA-0-0] [lt=26] is primary tenant, no need check readable_scn [2024-12-02 13:55:51.827579] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74A-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.860704] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.862368] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F8-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.862378] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.862393] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.862402] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.887278] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC53-0-0] [lt=12] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:51.889692] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC53-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:51.931320] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74B-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.950000] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADD-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:51.967367] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F9-0-0] [lt=22] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:51.968604] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:51.968615] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:51.968630] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F9-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:51.968638] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3F9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:51.972846] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA8-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:51.975180] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA8-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:52.003739] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC65-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:52.037632] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAD9-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.040515] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.055378] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.073510] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.074921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.074932] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.074947] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.074956] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.144240] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74D-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.179760] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.181249] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.181259] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.181275] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FB-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.181284] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.198534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC2-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.240205] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC71-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.247717] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74E-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.286234] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.287489] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.287497] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE0B-0-0] [lt=7] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:52.287500] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FC-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.287517] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.287523] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FC-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.315459] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC2-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.350007] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC53-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:52.351364] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF74F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.352571] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC53-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:52.352592] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC53-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:52.354281] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118950599646001, replayable_scn:1733118950599646001, readable_scn:1733118950599646001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118952351022, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118950599646001, replayable_scn:1733118950599646001, readable_scn:1733118950599646001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118952351022, ora_rowscn_=1733118951785936000, content_changed=true) [2024-12-02 13:55:52.354528] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:52.392400] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FD-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.393882] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.393893] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.393909] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.393918] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.429478] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118952351022, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118950599646001, replayable_scn:1733118950599646001, readable_scn:1733118950599646001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:52.454885] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF750-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.484006] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC74-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:52.498776] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.499979] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FE-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.499989] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.500005] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.500014] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FE-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.558788] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF751-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.604724] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.610119] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FF-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.610130] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.610145] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.610153] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF3FF-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.662506] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF752-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.715002] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF400-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.716398] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF400-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.716408] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF400-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.716424] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF400-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.716433] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF400-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.779545] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF753-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.793708] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDB-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118952602345001, v:0}}], last_dump_ts_=1733118947737183, force_dump=false) [2024-12-02 13:55:52.793733] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDB-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:52.800857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDB-0-0] [lt=29] is primary tenant, no need check readable_scn [2024-12-02 13:55:52.821296] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF401-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.822502] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF401-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.822513] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF401-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.822528] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF401-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.822538] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF401-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.876157] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118952873803, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118952873803, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:52.882940] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF754-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.894907] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC54-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:52.897354] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC54-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:52.927358] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF402-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:52.930031] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF402-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:52.930042] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF402-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:52.930057] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF402-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:52.930066] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF402-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:52.981841] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA9-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:52.984217] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAA9-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:52.985612] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF755-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.005062] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC66-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:53.031738] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99B-0-0] [lt=20] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=357, Holding Memory(MB)=61, Tenant Remain Memory(MB)=493, Usable Remain Memory(MB)=296, Last Update Limit Timestamp=1733118953031641, Is Updated=true) [2024-12-02 13:55:53.031917] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99B-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733118953031641000, start_id=1733118953031641000, end_id=1733118953031641001) [2024-12-02 13:55:53.033566] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99B-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118953031641000, v:0}, tenant_id=1001) [2024-12-02 13:55:53.034856] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF403-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.036072] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF403-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.036082] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF403-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.036098] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF403-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.036107] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF403-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.055456] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6E-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.089634] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF756-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.140991] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF404-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.142602] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF404-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.142612] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF404-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.142628] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF404-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.142634] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF404-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.193213] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF757-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.198613] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC3-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.240309] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC72-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.247698] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF405-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.248913] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF405-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.248923] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF405-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.248938] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF405-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.248947] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF405-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.296609] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF758-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.315538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC3-0-0] [lt=22] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.353962] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF406-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.354673] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:53.354692] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=268, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:53.355366] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC54-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:53.355585] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF406-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.355595] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF406-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.355611] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF406-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.355620] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF406-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.357925] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC54-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:53.357945] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC54-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:53.400381] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF759-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.460572] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF407-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.475006] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF407-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.475016] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF407-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.475031] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF407-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.475040] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF407-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.484078] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC75-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.503749] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.580070] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF408-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.581965] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF408-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.581977] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF408-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.581994] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF408-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.582003] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF408-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.607323] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.630448] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAD8-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:55:53.686959] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF409-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.709820] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF409-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.709831] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF409-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.709847] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF409-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.709852] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF409-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.710816] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.805896] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDC-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:53.811449] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDC-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:55:53.814508] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75D-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.814774] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75D-0-0] [lt=58] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118948789375, force_dump=false) [2024-12-02 13:55:53.814904] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.816332] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.816340] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40A-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.816355] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.816361] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40A-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.870764] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2324797][T1019_BackupDat][T1019][YB42C0A800AF-0006284313CEE95B-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:55:53.902459] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC55-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:53.904866] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC55-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:53.917554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.921303] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:53.921889] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:53.921900] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:53.921916] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:53.921925] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:53.930366] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE974-0-0] [lt=19] [RESTORE] restore scheduler quit [2024-12-02 13:55:53.931500] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE974-0-0] [lt=14] get targets(ret=0, jobs=[], exec_tenant_id=1, sql=select * from __all_recover_table_job) [2024-12-02 13:55:53.931537] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE975-0-0] [lt=12][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733118953931514, ref_timestamp=1733118893926955, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:55:53.931621] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE976-0-0] [lt=9] [RESTORE] restore scheduler start [2024-12-02 13:55:53.931627] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=5] [RESTORE] try process restore job [2024-12-02 13:55:53.932723] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:236) [2324796][T1019_BACKUP_SC][T1019][YB42C0A800AF-0006284313BEE94C-0-0] [lt=20] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-12-02 13:55:53.933013] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=1) [2024-12-02 13:55:53.934183] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=9][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:55:53.934193] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=8][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_restore_job_history where initiator_tenant_id = 1 AND initiator_job_id = 11, exec_tenant_id=1019) [2024-12-02 13:55:53.934260] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=6][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={initiator_tenant_id:1, initiator_job_id:11}) [2024-12-02 13:55:53.934271] WDIAG [SHARE] get_restore_job_history (ob_restore_persist_helper.cpp:827) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=10][errcode=-4018] failed to get persist info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", restore_key={initiator_tenant_id:1, initiator_job_id:11}) [2024-12-02 13:55:53.934277] WDIAG [RS.RESTORE] get_user_restore_job_history (ob_restore_util.cpp:1846) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=5][errcode=-4018] failed to get restore progress(ret=-4018, ret="OB_ENTRY_NOT_EXIST", initiator_job_id=11, initiator_tenant_id=1) [2024-12-02 13:55:53.934284] WDIAG [RS.RESTORE] restore_wait_tenant_finish (ob_restore_scheduler.cpp:1459) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=6][errcode=-4018] failed to get user restore job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", job_info={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:55:53.934310] INFO [RS.RESTORE] process_sys_restore_job (ob_restore_scheduler.cpp:169) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=21] [RESTORE] doing restore(ret=-4018, job={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:55:53.934325] WDIAG [RS.RESTORE] do_work (ob_restore_scheduler.cpp:121) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=15][errcode=-4018] failed to process sys restore job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", job_info={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:55:53.954903] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADE-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:53.990802] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAA-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:53.994234] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAA-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:54.006453] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC67-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:54.021094] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF75F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.026748] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.028995] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.029007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40C-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.029023] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.029032] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.037957] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADA-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.055533] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC6F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.125083] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF760-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.129351] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC4F-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:55:54.129377] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=17] start try disaster recovery [2024-12-02 13:55:54.129389] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:55:54.130652] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.134398] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.135403] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:54.135438] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=29] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.135451] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.135510] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=55] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:55:54.135524] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.135540] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=14] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.135575] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.135593] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.135604] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.135613] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.137765] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:55:54.137788] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:54.139083] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.143519] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=16] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:54.143542] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=18] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.143549] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.143612] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=58] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:55:54.143623] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.143645] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.143683] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=30] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.143702] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.143713] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.143723] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.144154] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40D-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.144166] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40D-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.144185] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40D-0-0] [lt=16][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.144196] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40D-0-0] [lt=10] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.145724] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:55:54.147064] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:54.148278] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.152646] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:54.152677] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.152689] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.152741] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:55:54.152754] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.152767] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.152801] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.152817] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.152826] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.152835] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.154614] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.161257] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:54.161286] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.161299] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.161351] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=49] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:55:54.161361] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.161374] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.161409] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.161424] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.161434] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.161443] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.163291] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:55:54.163312] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:54.164533] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.168738] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:54.168767] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.168779] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.168832] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=49] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:55:54.168844] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.168858] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.168891] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.168906] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.168916] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.168924] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.170714] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:55:54.171898] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:54.173061] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=20] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.177337] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:55:54.177367] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.177378] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.177430] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:55:54.177443] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.177456] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.177503] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=37] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.177524] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.177534] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.177545] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.179530] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:55:54.186159] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:55:54.186189] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.186200] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:55:54.186254] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=51] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:55:54.186263] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:55:54.186278] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:55:54.186311] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.186327] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.186336] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.186345] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:55:54.188342] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:55:54.188364] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=17] start check unit migrate finish [2024-12-02 13:55:54.188382] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:55:54.194160] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:55:54.198687] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.204284] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=21] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:55:54.214315] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC50-0-0] [lt=20] start to check alter locality finish [2024-12-02 13:55:54.215424] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC51-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.216546] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC51-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.218889] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC53-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.219998] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC53-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.222246] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC55-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.223340] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC55-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.224512] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC55-0-0] [lt=19] start check shrink resource pool [2024-12-02 13:55:54.224535] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC56-0-0] [lt=17] start check shrink resource pool(tenant_id=1) [2024-12-02 13:55:54.225584] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC56-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.226671] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC56-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.226748] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC57-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:55:54.227790] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC57-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.228875] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC57-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.228952] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC58-0-0] [lt=19] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:55:54.229323] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF761-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.230003] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC58-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.231091] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC58-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.231168] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC58-0-0] [lt=18] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.231185] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC58-0-0] [lt=15] start check primary zone [2024-12-02 13:55:54.231199] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC59-0-0] [lt=9] start check primary zone(tenant_id=1) [2024-12-02 13:55:54.232227] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC59-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.232302] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5A-0-0] [lt=18] start check primary zone(tenant_id=1002) [2024-12-02 13:55:54.233337] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5A-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.233411] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=16] start check primary zone(tenant_id=1020) [2024-12-02 13:55:54.234451] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:55:54.234527] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=17] start do unit balance [2024-12-02 13:55:54.235473] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=15] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118954235461}) [2024-12-02 13:55:54.235624] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320127][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE0C-0-0] [lt=26] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:54.236961] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=18] start distribute for server status change [2024-12-02 13:55:54.237016] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=19] finish distribute for server status change(ret=0) [2024-12-02 13:55:54.237028] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=11] finish do unit balance(ret=0) [2024-12-02 13:55:54.239357] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=9] start to do tenant group unit balance [2024-12-02 13:55:54.239399] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=19] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:55:54.239415] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:55:54.239435] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=15] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:55:54.239460] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:54.239481] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:54.239493] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:55:54.239506] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=9] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:54.239530] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:55:54.239570] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=39] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:55:54.239582] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:55:54.239598] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:55:54.239611] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:55:54.240412] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC73-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.249174] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40E-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.259133] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.259143] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.259158] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.259167] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.315612] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.333133] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF762-0-0] [lt=19] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118949323869, force_dump=false) [2024-12-02 13:55:54.333172] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF762-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.354593] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:54.360683] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC55-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:54.363315] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC55-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:54.363336] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC55-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:54.364106] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.365501] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.365512] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.365527] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.365536] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF40F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.436728] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF763-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.470412] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF410-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.472026] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF410-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.472037] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF410-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.472053] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF410-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.472061] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF410-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.484148] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC76-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:54.540296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF764-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.576981] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF411-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.578330] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF411-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.578337] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF411-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.578353] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF411-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.578363] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF411-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.643764] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF765-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.683183] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF412-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.684434] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF412-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.684445] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF412-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.684461] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF412-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.684466] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF412-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.747502] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF766-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.789315] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF413-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.790767] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF413-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.790778] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF413-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.790794] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF413-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.790803] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF413-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.816455] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDD-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:54.832459] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDD-0-0] [lt=27] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118954603741001, v:0}}], last_dump_ts_=1733118949767052, force_dump=false) [2024-12-02 13:55:54.832481] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDD-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:54.850913] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF767-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.895663] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF414-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:54.896963] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF414-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:54.896974] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF414-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:54.896990] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF414-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:54.896999] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF414-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:54.910120] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC56-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:54.912560] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC56-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:54.954484] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF768-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.000741] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAB-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:55.001914] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF415-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.003091] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAB-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:55.003725] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF415-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.003735] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF415-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.003751] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF415-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.003756] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF415-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.007829] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC68-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:55.055609] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC70-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.057770] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF769-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.108666] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF416-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.112036] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF416-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.112046] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF416-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.112062] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF416-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.112070] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF416-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.161526] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.198767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC5-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.216949] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF417-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.219025] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF417-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.219036] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF417-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.219053] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF417-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.219061] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF417-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.240526] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC74-0-0] [lt=54] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.261549] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99B-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733118955260537000, start_id=1733118955260537000, end_id=1733118955260537001) [2024-12-02 13:55:55.263230] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99B-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118955260537000, v:0}, tenant_id=1002) [2024-12-02 13:55:55.265269] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.315687] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC5-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.323914] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF418-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.325671] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF418-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.325681] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF418-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.325696] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF418-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.325702] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF418-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.354506] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118953603065001, replayable_scn:1733118953603065001, readable_scn:1733118953603065001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118955351081, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118953603065001, replayable_scn:1733118953603065001, readable_scn:1733118953603065001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118955351081, ora_rowscn_=1733118954818379001, content_changed=true) [2024-12-02 13:55:55.354749] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=20] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:55.366020] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC56-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:55.368559] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC56-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:55.368580] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC56-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:55.368669] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76C-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.430436] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF419-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.431102] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118955351081, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118953603065001, replayable_scn:1733118953603065001, readable_scn:1733118953603065001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:55.447852] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF419-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.447863] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF419-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.447876] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF419-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.447885] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF419-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.472741] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76D-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.484219] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC77-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.552792] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.554184] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.554196] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.554212] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41A-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.554221] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.576228] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76E-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.630877] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAD9-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:55:55.659110] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.661371] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41B-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.661382] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.661397] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.661407] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.680271] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF76F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.766230] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.767607] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.767614] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41C-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.767631] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.767636] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.783684] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF770-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.839439] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDE-0-0] [lt=4] is primary tenant, no need check readable_scn [2024-12-02 13:55:55.845039] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDE-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:55:55.872454] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.873719] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.873729] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.873745] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.873753] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.877737] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118955875227, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118955875227, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:55.887034] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF771-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.917833] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC57-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:55.920288] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC57-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:55.959782] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEADF-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:55.978607] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:55.979213] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:55.988601] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:55.988611] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:55.988627] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:55.988636] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:55.990672] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF772-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.009205] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC69-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:56.009813] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAC-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:56.012208] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAC-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:56.038279] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADB-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.055683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC71-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.093492] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.094946] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41F-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.094956] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.094971] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.094981] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF41F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.095037] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF773-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.133569] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:56.134794] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:55:56.134840] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:55:56.134847] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:55:56.134854] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:55:56.134858] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:55:56.134865] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:55:56.134869] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=3][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118961133583, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:56.197714] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF774-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.198849] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC6-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.199844] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF420-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.201407] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF420-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.201418] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF420-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.201434] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF420-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.201442] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF420-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.221562] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:56.222687] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956221499, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:55:56.222711] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961222709, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:55:56.240624] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC75-0-0] [lt=34] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.301530] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF775-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.306257] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF421-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.307513] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF421-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.307524] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF421-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.307540] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF421-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.307549] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF421-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.315762] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC6-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.316349] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:55:56.317465] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956316287, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:55:56.317500] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=32] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961317498, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:55:56.318592] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118956317434, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:55:56.318613] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961318612, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:55:56.354837] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:56.356139] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:55:56.371290] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC57-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:56.373833] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC57-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:56.373849] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC57-0-0] [lt=16] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:56.404787] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF776-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.412291] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF422-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.414732] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF422-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.414743] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF422-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.414759] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF422-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.414768] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF422-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.466468] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE0D-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-02 13:55:56.484293] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC78-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:56.508493] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF777-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.519705] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF423-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.520990] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF423-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.521001] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF423-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.521017] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF423-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.521022] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF423-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.612005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF778-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.625903] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF424-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.627121] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF424-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.627132] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF424-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.627147] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF424-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.627156] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF424-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.715612] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF779-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.732011] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF425-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.733485] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF425-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.733496] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF425-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.733512] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF425-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.733521] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF425-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.759491] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96A-0-0] [lt=27] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:56.818934] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.838296] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF426-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.839640] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF426-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.839650] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF426-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.839666] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF426-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.839675] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF426-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.850253] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDF-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:55:56.855611] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECDF-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:55:56.922339] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.925705] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC58-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:56.929307] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC58-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:56.944495] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF427-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:56.945912] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF427-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:56.945922] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF427-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:56.945938] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF427-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:56.945944] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF427-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:56.965507] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96A-0-0] [lt=30] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:56.966721] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96A-0-0] [lt=17] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:55:56.967903] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96A-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.010563] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6A-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:57.018924] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAD-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:57.021348] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAD-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:57.026092] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.050745] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF428-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.051943] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF428-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.051954] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF428-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.051970] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF428-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.051979] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF428-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.055757] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC72-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.129632] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.132589] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77D-0-0] [lt=51] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.156808] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF429-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.158032] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF429-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.158043] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF429-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.158055] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF429-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.158064] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF429-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.198929] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC7-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.232922] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.240728] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC76-0-0] [lt=42] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.262909] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42A-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.264149] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42A-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.264159] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.264175] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.264183] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.315840] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC7-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.336825] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF77F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.354870] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:57.354890] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=15] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=237, return_code_array=[0], broadcast_times=1) [2024-12-02 13:55:57.369020] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.370584] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.370595] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.370611] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.370619] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.376524] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC58-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:57.379077] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC58-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:57.379095] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC58-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:57.422231] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=16] start checking lost replicas [2024-12-02 13:55:57.440907] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF780-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.475524] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.476736] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.476750] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42C-0-0] [lt=13][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.476766] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.476775] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.484363] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC79-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.544863] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF781-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.581689] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.583500] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.583510] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.583526] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.583531] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42D-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.631324] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEADA-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:57.648594] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF782-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.688486] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.689066] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.689077] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.689092] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.689101] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.752004] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF783-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.755255] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF783-0-0] [lt=52] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.793997] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.796086] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.796096] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.796112] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.796120] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF42F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.855754] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF784-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.860460] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE0-0-0] [lt=15] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118957607302002, v:0}}], last_dump_ts_=1733118952793706, force_dump=false) [2024-12-02 13:55:57.860484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE0-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:55:57.865902] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE0-0-0] [lt=23] is primary tenant, no need check readable_scn [2024-12-02 13:55:57.900981] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF430-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:57.902509] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF430-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.902520] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF430-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:57.902535] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF430-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:57.902544] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF430-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:57.934626] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC59-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:57.937090] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC59-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:57.964527] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE0-0-0] [lt=24] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:57.966129] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF785-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:57.967412] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=0] start do ObArchiveSchedulerService round(round=18) [2024-12-02 13:55:57.969631] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:55:57.969651] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=18][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2024-12-02 13:55:57.969712] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=7][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:57.969725] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=13][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:55:57.969732] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95D-0-0] [lt=7][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:55:58.007441] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF431-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.008757] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF431-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.008767] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF431-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.008783] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF431-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.008792] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF431-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.011940] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6B-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:58.028056] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAE-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:58.030477] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAE-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:58.038596] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADC-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.055828] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC73-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.071313] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF786-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.113674] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF432-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.115345] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF432-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.115356] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF432-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.115372] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF432-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.115382] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF432-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.139304] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=13] failed to get result(ret=-4008) [2024-12-02 13:55:58.175222] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF787-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.199009] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC8-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.220341] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF433-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.221713] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF433-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.221723] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF433-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.221740] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF433-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.221750] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF433-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.240826] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC77-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.279256] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF788-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.286276] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE10-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:58.288944] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320103][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE11-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:55:58.289143] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320114][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE12-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:55:58.289883] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE13-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:55:58.290106] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE14-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:55:58.290918] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320116][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE1A-0-0] [lt=3] [DDL] execute ddl like stmt(ret=0, cost=29, ddl_arg=NULL) [2024-12-02 13:55:58.291505] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE1B-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:55:58.315918] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.326799] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF434-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.328403] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF434-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.328414] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF434-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.328430] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF434-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.328436] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF434-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.354775] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118956756133000, replayable_scn:1733118956756133000, readable_scn:1733118956756133000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118958351095, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118956756133000, replayable_scn:1733118956756133000, readable_scn:1733118956756133000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118958351095, ora_rowscn_=1733118957862799000, content_changed=true) [2024-12-02 13:55:58.355025] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:58.381820] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC59-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:58.383095] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF789-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.384336] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC59-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:58.384357] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC59-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:58.432759] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=11] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118958351095, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118956756133000, replayable_scn:1733118956756133000, readable_scn:1733118956756133000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:55:58.433495] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF435-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.434707] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF435-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.434720] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF435-0-0] [lt=12][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.434737] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF435-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.434746] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF435-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.469440] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE1D-0-0] [lt=4] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:55:58.484432] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7A-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:58.486128] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.539823] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF436-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.541355] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF436-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.541366] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF436-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.541383] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF436-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.541392] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF436-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.589925] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.646527] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF437-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.666510] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF437-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.666521] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF437-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.666537] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF437-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.666545] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF437-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.693366] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.771588] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF438-0-0] [lt=32] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.772827] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF438-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.772838] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF438-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.772854] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF438-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.772863] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF438-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.797299] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.871353] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE1-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:55:58.877767] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE1-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:55:58.878097] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF439-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.879412] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118958877028, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118958877028, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:55:58.880890] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF439-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.880900] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF439-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.880917] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF439-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.880926] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF439-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:58.901017] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.901262] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78E-0-0] [lt=54] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118953814773, force_dump=false) [2024-12-02 13:55:58.942631] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5A-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:58.945106] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5A-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:58.985878] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:58.987546] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:58.987557] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:58.987573] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:58.987579] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43A-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.004538] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF78F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.013320] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6C-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:59.037156] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAF-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:59.039564] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAAF-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:59.055903] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC74-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:59.092427] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.093889] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.093900] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.093916] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.093924] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.108031] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF790-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.198793] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.199083] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAC9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:59.201006] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.201016] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.201032] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.201041] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.211438] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF791-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.240927] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC78-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:59.305936] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.307592] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43D-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.307603] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.307619] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.307628] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.315109] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF792-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.315991] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAC9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:59.342993] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99C-0-0] [lt=17] get number(ret=0, service_type_=0, range=1, base_id=1733118959342901000, start_id=1733118959342901000, end_id=1733118959342901001) [2024-12-02 13:55:59.345421] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99C-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118959342901000, v:0}, tenant_id=1) [2024-12-02 13:55:59.355002] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:55:59.387080] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5A-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:59.389632] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5A-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:55:59.389654] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5A-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:55:59.412485] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.413763] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.413773] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.413786] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.413795] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.418853] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF793-0-0] [lt=52] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118954333131, force_dump=false) [2024-12-02 13:55:59.418893] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF793-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.484509] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7B-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:55:59.518701] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.520381] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.520388] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43F-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.520404] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.520413] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF43F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.522236] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF794-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.625272] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF440-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.625874] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF795-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.626602] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF440-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.626613] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF440-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.626629] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF440-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.626638] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF440-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.631741] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEADB-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:55:59.729172] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF796-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.731501] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF441-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.733293] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF441-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.733304] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF441-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.733320] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF441-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.733329] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF441-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.833364] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF797-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.838154] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF442-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.839665] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF442-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.839676] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF442-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.839692] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF442-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.839702] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF442-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.883425] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE2-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:55:59.890473] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE2-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118959609045002, v:0}}], last_dump_ts_=1733118954832457, force_dump=false) [2024-12-02 13:55:59.890496] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE2-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:55:59.936794] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF798-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.944551] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF443-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:55:59.945930] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF443-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:55:59.945940] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF443-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:55:59.945956] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF443-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:55:59.945965] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF443-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:55:59.950528] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5B-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:55:59.952999] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5B-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:55:59.969234] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE1-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.014713] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6D-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:00.038918] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADD-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.040137] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF799-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.046272] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB0-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:00.048728] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB0-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:00.050813] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF444-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.052714] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF444-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.052725] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF444-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.052741] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF444-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.052750] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF444-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.055974] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC75-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.144566] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79A-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.157678] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF445-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.159170] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF445-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.159180] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF445-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.159197] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF445-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.159206] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF445-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.199158] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.241045] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC79-0-0] [lt=48] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.248944] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79B-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.264041] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF446-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.265551] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF446-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.265561] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF446-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.265578] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF446-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.265584] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF446-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.316069] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACA-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.352704] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79C-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.355087] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:00.370526] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF447-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.371958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF447-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.371968] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF447-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.371985] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF447-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.371994] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF447-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.392284] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5B-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:00.394803] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5B-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:00.394822] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5B-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:00.456237] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79D-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.476968] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF448-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.478233] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF448-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.478244] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF448-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.478260] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF448-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.478269] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF448-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.484584] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7C-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:00.530769] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=10] start do ObArchiveSchedulerService round(round=18) [2024-12-02 13:56:00.534236] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=20][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:56:00.534260] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=23][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-12-02 13:56:00.534324] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:00.534342] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=18][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:56:00.534353] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95D-0-0] [lt=10][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:00.559556] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79E-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.583237] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF449-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.585013] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF449-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.585024] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF449-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.585039] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF449-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.585048] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF449-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.663272] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF79F-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.690065] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.691927] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.691938] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.691955] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44A-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.691964] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.767455] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A0-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.796892] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.799054] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44B-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.799064] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44B-0-0] [lt=10][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:00.799081] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44B-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.871291] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.895324] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE3-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:56:00.900623] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE3-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:00.904181] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:00.905437] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.905448] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:00.905464] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:00.905483] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44C-0-0] [lt=17] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:00.958345] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5C-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:00.960799] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5C-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:00.975057] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:00.981735] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:00.995136] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=19] failed to get result(ret=-4008) [2024-12-02 13:56:01.010317] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.012079] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.012090] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.012105] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.012114] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.016081] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6E-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:01.055449] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB1-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:01.056046] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC76-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.057867] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB1-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:01.078465] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.117075] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.118553] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.118561] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44E-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.118574] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.118583] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.133649] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:01.135060] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:01.135117] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:01.135130] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:01.135141] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:01.135150] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:01.135161] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:01.135169] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118966133668, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:01.181883] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A4-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.199235] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACB-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.222772] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:01.223561] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.223943] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961222709, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:01.223969] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966223966, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:01.225308] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.225320] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44F-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.225336] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.225345] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF44F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.241151] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7A-0-0] [lt=43] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.248452] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE972-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733118961248395000, start_id=1733118961248395000, end_id=1733118961248395001) [2024-12-02 13:56:01.250403] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE972-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118961248395000, v:0}, tenant_id=1019) [2024-12-02 13:56:01.285390] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.316138] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACB-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.317562] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:01.318719] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961317498, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:01.318745] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966318742, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:01.319854] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118961318612, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:01.319877] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966319875, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:01.330319] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF450-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.332055] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF450-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.332065] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF450-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.332082] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF450-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.332091] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF450-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.354867] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118959609045002, replayable_scn:1733118959609045002, readable_scn:1733118959609045002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118961351549, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118959609045002, replayable_scn:1733118959609045002, readable_scn:1733118959609045002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118961351549, ora_rowscn_=1733118960898138000, content_changed=true) [2024-12-02 13:56:01.355123] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:01.355138] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=248, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:01.356438] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:01.388830] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A6-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.397451] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:01.399968] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5C-0-0] [lt=76] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:01.399988] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5C-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:01.434339] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=16] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118961351549, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118959609045002, replayable_scn:1733118959609045002, readable_scn:1733118959609045002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:01.437030] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF451-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.438594] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF451-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.438605] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF451-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.438621] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF451-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.438630] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF451-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.484657] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.492503] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.543637] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF452-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.544918] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF452-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.544927] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF452-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.544944] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF452-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.544954] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF452-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.595819] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A8-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.599606] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:56:01.632187] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEADC-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:01.649939] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF453-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.651175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF453-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.651186] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF453-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.651198] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF453-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.651208] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF453-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.700096] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7A9-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.756267] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF454-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.772301] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF454-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.772311] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF454-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.772328] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF454-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.772337] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF454-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.803474] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AA-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.877416] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF455-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.879996] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF455-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.880007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF455-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.880023] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF455-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.880032] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF455-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:01.881021] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118961877691, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118961877691, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:01.905749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE4-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:01.907238] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.911537] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE4-0-0] [lt=26] is primary tenant, no need check readable_scn [2024-12-02 13:56:01.966128] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5D-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:01.968571] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5D-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:01.974132] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE2-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:01.985042] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF456-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:01.986657] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF456-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:01.986668] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF456-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:01.986684] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF456-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:01.986692] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF456-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.010420] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.017490] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC6F-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:02.039226] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADE-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.056118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC77-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.064594] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB2-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:02.067016] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB2-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:02.091627] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF457-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.093153] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF457-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.093164] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF457-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.093179] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF457-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.093188] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF457-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.113728] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AD-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.198155] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF458-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.199311] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.200028] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF458-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.200038] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF458-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.200054] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF458-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.200063] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF458-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.217205] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AE-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.241264] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7B-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.305069] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF459-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.306320] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF459-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.306331] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF459-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.306347] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF459-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.306356] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF459-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.316211] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACC-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.320575] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7AF-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.355105] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:02.402659] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5D-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:02.405171] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5D-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:02.405190] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5D-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:02.411350] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.413187] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.413198] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.413213] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.413222] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.424112] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B0-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.484732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:02.518247] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45B-0-0] [lt=22] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.519608] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.519619] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.519635] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.519644] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45B-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.527324] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B1-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.624626] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.625844] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45C-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.625855] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45C-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.625871] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45C-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.625880] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.630924] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.730974] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.733193] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.733204] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.733221] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.733229] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.734412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.837973] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B4-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.838123] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.845407] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.845418] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.845430] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.845435] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.916963] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE5-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118962612193003, v:0}}], last_dump_ts_=1733118957860458, force_dump=false) [2024-12-02 13:56:02.916983] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE5-0-0] [lt=16] is primary tenant, no need check readable_scn [2024-12-02 13:56:02.922621] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE5-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:56:02.946994] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.950311] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45F-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:02.951460] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:02.951470] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:02.951487] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:02.951496] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF45F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:02.973958] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5E-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:02.976430] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5E-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:03.018883] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC70-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:03.039954] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99C-0-0] [lt=18] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=357, Holding Memory(MB)=61, Tenant Remain Memory(MB)=493, Usable Remain Memory(MB)=296, Last Update Limit Timestamp=1733118963039352, Is Updated=true) [2024-12-02 13:56:03.040148] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99C-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1733118963039352000, start_id=1733118963039352000, end_id=1733118963039352001) [2024-12-02 13:56:03.042254] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99C-0-0] [lt=17] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118963039352000, v:0}, tenant_id=1001) [2024-12-02 13:56:03.050338] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B6-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.056193] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC78-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.056391] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF460-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.057894] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF460-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.057904] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF460-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.057920] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF460-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.057930] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF460-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.073635] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB3-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:03.076013] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB3-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:03.154248] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B7-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.162766] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF461-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.163969] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF461-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.163980] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF461-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.163995] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF461-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.164004] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF461-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.199385] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACD-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.241371] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7C-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.257756] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B8-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.268890] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF462-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.270384] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF462-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.270395] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF462-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.270411] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF462-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.270420] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF462-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.316285] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.355096] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:03.361619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7B9-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.375335] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF463-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.391800] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF463-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.391809] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF463-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.391829] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF463-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.391837] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF463-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.407823] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5E-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:03.410335] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5E-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:03.410356] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5E-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:03.464950] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BA-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.484807] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC7F-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.496782] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF464-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.498212] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF464-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.498223] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF464-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.498239] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF464-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.498248] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF464-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.585101] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.603255] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF465-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.605180] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF465-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.605190] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF465-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.605208] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF465-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.605218] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF465-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.632633] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEADD-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:03.634576] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=17] failed to get result(ret=-4008) [2024-12-02 13:56:03.688641] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.710117] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF466-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.711400] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF466-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.711410] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF466-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.711425] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF466-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.711433] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF466-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.792254] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BD-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.816249] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF467-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.817736] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF467-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.817747] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF467-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.817761] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF467-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.817766] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF467-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.895504] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BE-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.922679] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF468-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:03.924111] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF468-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:03.924119] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF468-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:03.924134] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF468-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:03.924142] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF468-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:03.927484] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE6-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:03.934084] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE6-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:03.978962] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE3-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:03.981947] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5F-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:03.984421] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC5F-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:04.009529] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.009797] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7BF-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118958901261, force_dump=false) [2024-12-02 13:56:04.020256] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC71-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:04.029072] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF469-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.030293] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF469-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.030304] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF469-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.030320] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF469-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.030332] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF469-0-0] [lt=11] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.039549] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEADF-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.056271] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC79-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.082715] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB4-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:04.086204] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB4-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:04.115662] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.135169] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46A-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.137011] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46A-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.199458] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACE-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.218928] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.239189] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5B-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:56:04.239208] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] start try disaster recovery [2024-12-02 13:56:04.239220] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:04.240449] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.241465] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7D-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.242116] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46B-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.244144] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46B-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.244155] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.244171] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.244180] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.245018] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:04.245046] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.245058] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.245116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=54] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:04.245129] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.245143] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.245177] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.245194] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.245203] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.245212] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.247264] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:04.247285] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=20] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:04.248562] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.252994] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:04.253023] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.253035] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.253089] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=50] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:04.253103] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.253117] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.253153] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.253169] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.253179] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.253188] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.254984] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:04.256284] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=17] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:04.257485] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=19] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.261787] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:04.261817] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.261829] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=11] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.261880] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=49] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:04.261893] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.261906] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.261941] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.261956] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.261966] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.261975] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.263787] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.270360] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:04.270386] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.270397] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.270448] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=48] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:04.270460] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.270474] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.270507] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.270523] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.270532] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.270541] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.272550] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:04.272565] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:04.273769] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.278023] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:04.278049] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.278060] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.278112] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=48] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:04.278125] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.278139] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.278175] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.278191] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.278201] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.278210] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.280027] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:04.281151] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:04.282380] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=19] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.286598] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:04.286625] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.286637] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.286689] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:04.286702] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.286715] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.286751] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.286767] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.286776] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.286784] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.288742] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:04.295403] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:04.295433] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.295442] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:04.295495] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:04.295505] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:04.295518] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:04.295553] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.295567] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.295577] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.295585] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:04.297550] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:04.297573] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=19] start check unit migrate finish [2024-12-02 13:56:04.297585] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:04.303385] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:04.313592] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=18] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:04.316360] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.323164] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C2-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.323971] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5C-0-0] [lt=17] start to check alter locality finish [2024-12-02 13:56:04.325102] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5D-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.326247] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5D-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.328547] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5F-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.329682] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC5F-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.331987] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC61-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.333100] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC61-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.334317] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC61-0-0] [lt=21] start check shrink resource pool [2024-12-02 13:56:04.334339] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC62-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:04.335404] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC62-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.336512] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC62-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.336589] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC63-0-0] [lt=16] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:04.337657] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC63-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.338760] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC63-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.338837] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC64-0-0] [lt=17] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:04.339894] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC64-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.341009] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC64-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.341087] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC64-0-0] [lt=16] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.341103] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC64-0-0] [lt=15] start check primary zone [2024-12-02 13:56:04.341113] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC65-0-0] [lt=5] start check primary zone(tenant_id=1) [2024-12-02 13:56:04.342172] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC65-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.342249] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC66-0-0] [lt=17] start check primary zone(tenant_id=1002) [2024-12-02 13:56:04.343297] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC66-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.343372] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=17] start check primary zone(tenant_id=1020) [2024-12-02 13:56:04.344424] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:04.344500] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=17] start do unit balance [2024-12-02 13:56:04.345491] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC67-0-0] [lt=14] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118964345478}) [2024-12-02 13:56:04.345687] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE1F-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:04.347062] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=18] start distribute for server status change [2024-12-02 13:56:04.347118] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=16] finish distribute for server status change(ret=0) [2024-12-02 13:56:04.347129] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:56:04.349244] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.349505] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=9] start to do tenant group unit balance [2024-12-02 13:56:04.349547] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=17] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:04.349563] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=11] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:04.349583] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=14] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:04.349609] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=14] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:04.349632] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:04.349645] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:04.349657] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=9] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:04.349681] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:04.349718] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=37] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:04.349730] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:04.349745] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:04.349758] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:04.350615] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.350622] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46C-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.350636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.350642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.355008] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118962612193003, replayable_scn:1733118962612193003, readable_scn:1733118962612193003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118964352367, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118962612193003, replayable_scn:1733118962612193003, readable_scn:1733118962612193003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118964352367, ora_rowscn_=1733118963929421002, content_changed=true) [2024-12-02 13:56:04.355243] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:04.413059] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:04.415624] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5F-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:04.415646] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC5F-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:04.427976] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C3-0-0] [lt=11] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118959418851, force_dump=false) [2024-12-02 13:56:04.428017] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C3-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.435922] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118964352367, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118962612193003, replayable_scn:1733118962612193003, readable_scn:1733118962612193003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:04.455637] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.457294] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46D-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.457303] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46D-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.457315] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46D-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.457323] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.484881] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC80-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:04.531686] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C4-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.562324] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.563804] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.563814] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.563829] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.563836] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46E-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.635680] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.668735] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.670002] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.670012] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.670028] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.670036] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF46F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.739682] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C6-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.775032] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF470-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.776676] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF470-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.776686] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF470-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.776701] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF470-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.776709] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF470-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.843116] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C7-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.881624] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF471-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.882657] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118964879397, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118964879397, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:04.882917] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF471-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.882926] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF471-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.882941] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF471-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.882948] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF471-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.939148] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:04.945064] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE7-0-0] [lt=11] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118964614352003, v:0}}], last_dump_ts_=1733118959890472, force_dump=false) [2024-12-02 13:56:04.945087] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE7-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:04.946689] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.987895] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF472-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:04.989508] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF472-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:04.989514] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF472-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:04.989529] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF472-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:04.989535] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF472-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:04.989779] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC60-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:04.992239] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC60-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:05.021647] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC72-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:05.050331] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7C9-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.056348] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7A-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.092945] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB5-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:05.094395] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF473-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.095390] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB5-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:05.095791] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF473-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.095798] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF473-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.095810] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF473-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.095815] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF473-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.153755] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CA-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.199534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEACF-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.200799] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF474-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.202375] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF474-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.202385] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF474-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.202401] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF474-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.202411] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF474-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.241582] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7E-0-0] [lt=50] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.257991] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.269151] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99C-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733118965268793000, start_id=1733118965268793000, end_id=1733118965268793001) [2024-12-02 13:56:05.270765] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99C-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118965268793000, v:0}, tenant_id=1002) [2024-12-02 13:56:05.307365] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF475-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.308993] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF475-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.309004] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF475-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.309020] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF475-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.309029] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF475-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.316433] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEACF-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.355336] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:05.355350] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=249, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:05.361397] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CC-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.413987] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF476-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.418366] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC60-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:05.420904] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC60-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:05.420925] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC60-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:05.468456] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF476-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.468465] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF476-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.468478] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF476-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.468484] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF476-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.484960] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC81-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.546901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CD-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.573402] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF477-0-0] [lt=15] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.574717] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF477-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.574728] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF477-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.574744] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF477-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.574750] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF477-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.633090] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEADE-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:56:05.650342] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.679640] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF478-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.681108] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF478-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.681118] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF478-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.681134] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF478-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.681142] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF478-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.753944] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7CF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.786022] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF479-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.787692] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF479-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.787702] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF479-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.787717] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF479-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.787726] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF479-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.857400] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.892604] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.893145] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.893156] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.893171] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.893180] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:05.950777] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE8-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:56:05.956841] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE8-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:56:05.961128] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D1-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.983711] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE4-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:05.984396] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:05.997570] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC61-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:05.998090] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:05.999930] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:05.999941] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:05.999954] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47B-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:05.999958] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47B-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.000008] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC61-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:06.023001] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC73-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:06.039888] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE0-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.056424] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.064581] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.102304] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB6-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:06.104760] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB6-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:06.104813] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.106069] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47C-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.106079] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.106095] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.106104] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.133733] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:06.135001] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:06.135054] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:06.135061] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:06.135075] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:06.135080] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:06.135088] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:06.135093] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118971133749, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:06.168261] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D3-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.199611] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD0-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.210976] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.212288] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.212299] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.212313] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.212322] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.224033] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:06.225189] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966223966, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:06.225213] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971225211, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:06.241680] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC7F-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.271846] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D4-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.316507] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.317217] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.318808] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:06.319115] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.319126] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.319142] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.319147] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47E-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.319946] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966318742, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:06.319969] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971319967, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:06.321074] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118966319875, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:06.321095] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971321093, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:06.355358] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:06.356666] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:06.375397] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D5-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.423636] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC61-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:06.424035] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.425552] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.425563] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.425578] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.425583] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF47F-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.426227] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC61-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:06.426249] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC61-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:06.478793] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.485034] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC82-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:06.530436] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF480-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.532005] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF480-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.532012] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF480-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.532027] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF480-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.532037] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF480-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.582486] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.637060] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF481-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.651749] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF481-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.651759] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF481-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.651776] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF481-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.651784] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF481-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.685830] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.756709] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF482-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.757996] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF482-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.758007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF482-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.758023] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF482-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.758032] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF482-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.759580] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96B-0-0] [lt=26] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:06.789666] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7D9-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.862980] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF483-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.864537] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF483-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.864548] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF483-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.864565] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF483-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.864574] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF483-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.892925] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.961837] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE9-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:06.968053] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECE9-0-0] [lt=30] is primary tenant, no need check readable_scn [2024-12-02 13:56:06.969554] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF484-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:06.971585] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF484-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:06.971597] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF484-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:06.971609] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF484-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:06.971614] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF484-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.972048] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96B-0-0] [lt=32] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:06.973229] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96B-0-0] [lt=16] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:06.974388] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96B-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:06.996245] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DB-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.005351] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC62-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:07.006517] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:361) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC62-0-0] [lt=17] log restore source is empty, just skip(ret=-4018, user_tenant_id=1002) [2024-12-02 13:56:07.008969] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC62-0-0] [lt=11] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:07.024374] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC74-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:07.056500] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7C-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:07.076611] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF485-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.077834] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF485-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.077844] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF485-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.077860] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF485-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.077868] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF485-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.100995] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DC-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.111593] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB7-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:07.114061] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB7-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:07.182712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF486-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.184323] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF486-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.184333] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF486-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.184349] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF486-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.184358] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF486-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.199683] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:07.204523] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.207868] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DD-0-0] [lt=53] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.241775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC80-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:07.289262] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF487-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.290676] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF487-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.290686] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF487-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.290702] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF487-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.290711] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF487-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.308246] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.316576] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD1-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:07.355219] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118965615034003, replayable_scn:1733118965615034003, readable_scn:1733118965615034003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118967351612, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118965615034003, replayable_scn:1733118965615034003, readable_scn:1733118965615034003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118967351612, ora_rowscn_=1733118966964218000, content_changed=true) [2024-12-02 13:56:07.355461] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:07.395617] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF488-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.397862] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF488-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.397868] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF488-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.397884] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF488-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.397893] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF488-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.411555] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7DF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.426021] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=15] start checking lost replicas [2024-12-02 13:56:07.428905] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC62-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:07.431410] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC62-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:07.431429] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC62-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:07.437548] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118967351612, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118965615034003, replayable_scn:1733118965615034003, readable_scn:1733118965615034003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:07.469585] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE21-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:56:07.485109] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC83-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:07.502844] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF489-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.504053] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF489-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.504064] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF489-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.504079] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF489-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.504088] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF489-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.515332] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E0-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.608931] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.610214] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.610226] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48A-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.610241] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.610246] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48A-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.618658] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E1-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.633526] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEADF-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:56:07.715159] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.716378] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.716388] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.716404] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.716413] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.727980] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E2-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.821219] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.822598] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.822609] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.822626] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48C-0-0] [lt=16][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.822636] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.831302] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E3-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.845192] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E3-0-0] [lt=53] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.884154] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118967880928, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118967880928, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:07.920860] INFO [RS] prepare (ob_mview_maintenance_task.cpp:178) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=24] mview maintenance task prepare success(tenant_id_=1, round_=11, prepare_cost_us_=1) [2024-12-02 13:56:07.927758] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:07.929055] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.929062] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48D-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:07.929078] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:07.929084] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48D-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:07.941995] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=13] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:56:07.945657] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E4-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:07.972490] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2319871][T1_BackupDataSr][T1][YB42C0A800AF-0006284308DEE968-0-0] [lt=16] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:07.973756] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEA-0-0] [lt=15] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118967896725000, v:0}}], last_dump_ts_=1733118962916961, force_dump=false) [2024-12-02 13:56:07.973781] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEA-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:07.979566] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEA-0-0] [lt=30] is primary tenant, no need check readable_scn [2024-12-02 13:56:07.988499] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE5-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.014350] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC63-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:08.016823] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC63-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:08.025770] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC75-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:08.034109] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.036434] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.036442] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48E-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.036458] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.036467] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.040207] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE1-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.049130] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E5-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.056572] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.120897] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB8-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:08.123334] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB8-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:08.141387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48F-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.152450] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E6-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.162072] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.162082] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.162099] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.162107] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF48F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.199759] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.241866] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC81-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.263683] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.267246] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF490-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.268470] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF490-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.268481] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF490-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.268498] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF490-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.268504] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF490-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.316649] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD2-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.355469] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:08.355486] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=243, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:08.367509] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.373904] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF491-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.375207] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF491-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.375218] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF491-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.375235] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF491-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.375243] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF491-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.434117] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC63-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:08.436600] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC63-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:08.436618] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC63-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:08.472033] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7E9-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.480312] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF492-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.481664] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF492-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.481674] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF492-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.481690] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF492-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.481700] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF492-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.485183] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC84-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:08.543750] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE24-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:08.575031] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EA-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.586678] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF493-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.588243] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF493-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.588254] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF493-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.588270] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF493-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.588279] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF493-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.678595] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.693310] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF494-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.693904] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF494-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.693915] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF494-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.693931] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF494-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.693939] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF494-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.781963] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EC-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.798895] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF495-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.803371] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF495-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.803382] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF495-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.803398] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF495-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.803404] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF495-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.902532] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7ED-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.908426] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF496-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:08.909641] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF496-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:08.909653] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF496-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:08.909669] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF496-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:08.909679] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF496-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:08.985326] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEB-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:08.991685] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEB-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:09.007927] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.014742] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF497-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.015996] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF497-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.016006] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF497-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.016021] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF497-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.016030] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF497-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.022108] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC64-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:09.024546] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC64-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:09.027169] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC76-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:09.056650] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7E-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.111525] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EF-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.111794] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7EF-0-0] [lt=56] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118964009796, force_dump=false) [2024-12-02 13:56:09.121029] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF498-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.122580] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF498-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.122590] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF498-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.122606] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF498-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.122612] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF498-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.130115] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB9-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:09.132506] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAB9-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:09.199831] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD3-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.215266] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F0-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.227429] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF499-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.229587] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF499-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.229597] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF499-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.229613] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF499-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.229622] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF499-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.241962] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC82-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.316719] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD3-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.319689] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.334630] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.337045] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.337056] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.337072] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.337081] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49A-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.350942] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99D-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733118969350867000, start_id=1733118969350867000, end_id=1733118969350867001) [2024-12-02 13:56:09.352979] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99D-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118969350867000, v:0}, tenant_id=1) [2024-12-02 13:56:09.355496] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:09.423109] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F2-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.439283] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC64-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:09.441813] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC64-0-0] [lt=50] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:09.441830] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC64-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:09.442061] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.443373] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.443384] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.443401] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49B-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.443409] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.485258] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC85-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.526645] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F3-0-0] [lt=20] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118964427974, force_dump=false) [2024-12-02 13:56:09.526683] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F3-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.548356] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.550078] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.550089] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.550105] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49C-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.550114] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.630801] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.633948] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAE0-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:09.655024] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.668329] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.668339] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.668351] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49D-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.668360] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.767567] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F5-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.773326] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.774830] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.774841] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.774857] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.774866] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.871887] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F6-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.879792] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.881044] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.881054] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.881069] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.881078] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF49F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.975706] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.986023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:09.987573] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:09.987583] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:09.987599] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:09.987608] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A0-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:09.993291] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE6-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:09.997364] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEC-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:10.008379] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEC-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118969619236003, v:0}}], last_dump_ts_=1733118964945062, force_dump=false) [2024-12-02 13:56:10.008402] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEC-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:10.028529] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC77-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:10.029885] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC65-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:10.032326] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC65-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:10.040513] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE2-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.056726] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC7F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.079444] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.092539] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.094164] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.094174] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.094190] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.094198] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.139150] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABA-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:10.141516] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABA-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:10.182827] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7F9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.199057] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.199906] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.200873] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A2-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.200880] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A2-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.200891] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A2-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.200900] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.242052] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC83-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.286627] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.305887] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.307131] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A3-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.307141] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A3-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.307156] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.307164] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A3-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.316793] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.355306] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118968618533003, replayable_scn:1733118968618533003, readable_scn:1733118968618533003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118970352420, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118968618533003, replayable_scn:1733118968618533003, readable_scn:1733118968618533003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118970352420, ora_rowscn_=1733118970000156000, content_changed=true) [2024-12-02 13:56:10.355555] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:10.390402] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.412150] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.413394] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A4-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.413404] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.413419] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.413427] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A4-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.439110] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=12] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118970352420, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118968618533003, replayable_scn:1733118968618533003, readable_scn:1733118968618533003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:10.444508] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC65-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:10.447065] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC65-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:10.447087] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC65-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:10.485329] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC86-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:10.494951] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FC-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.518416] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A5-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.519696] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A5-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.519706] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.519721] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A5-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.519729] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A5-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.598926] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FD-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.624662] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A6-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.625911] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A6-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.625921] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.625936] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.625941] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.703078] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FE-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.730893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.732278] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A7-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.732288] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.732304] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.732312] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A7-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.806466] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF7FF-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.837205] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.838959] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A8-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.838968] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.838984] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A8-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.838991] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A8-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.885788] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118970882524, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118970882524, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:10.910020] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF800-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.943898] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A9-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:10.945635] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A9-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:10.945645] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A9-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:10.945660] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:10.945668] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4A9-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:10.987050] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:11.013618] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF801-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.013826] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECED-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:56:11.029181] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECED-0-0] [lt=16] is primary tenant, no need check readable_scn [2024-12-02 13:56:11.029935] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC78-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:11.037655] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC66-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:11.040097] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC66-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:11.050540] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.051768] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AA-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.051778] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.051795] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.051804] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.056802] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC80-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:11.117849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF802-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.133815] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:11.135209] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:11.135266] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:11.135278] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:11.135289] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:11.135297] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:11.135307] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:11.135315] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118976133829, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:11.148117] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABB-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:11.150497] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABB-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:11.156699] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.158253] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.158264] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.158280] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.158289] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.199981] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD5-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:11.221543] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF803-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.225275] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:11.226453] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971225211, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:11.226476] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976226473, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:11.242143] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC84-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:11.263345] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.264681] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.264693] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.264709] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.264718] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AC-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.275649] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE973-0-0] [lt=25] get number(ret=0, service_type_=0, range=1, base_id=1733118971274943000, start_id=1733118971274943000, end_id=1733118971274943001) [2024-12-02 13:56:11.277240] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE973-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118971274943000, v:0}, tenant_id=1019) [2024-12-02 13:56:11.316866] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:11.320030] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:11.321179] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971319967, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:11.321203] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976321201, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:11.322345] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118971321093, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:11.322368] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976322366, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:11.325219] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF804-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.355602] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:11.356884] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:11.369652] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.370878] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.370888] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AD-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.370904] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.370913] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.431140] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF805-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.449762] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC66-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:11.452273] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC66-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:11.452286] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC66-0-0] [lt=13] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:11.475866] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AE-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.477335] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.477346] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.477362] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AE-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.477371] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AE-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.485408] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC87-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:11.534627] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF806-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.582347] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.583921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.583932] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.583947] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.583956] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4AF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.634385] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAE1-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:11.638124] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF807-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.688999] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.690594] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.690605] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.690620] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.690626] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B0-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.741572] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF808-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.795525] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.796931] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.796941] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:11.796956] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:11.796965] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:11.844974] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF809-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.901892] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:11.962502] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:11.998067] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE7-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.031337] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC79-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:12.040824] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE3-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.045314] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC67-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:12.047818] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC67-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:12.056869] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC81-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.157238] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABC-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:12.159658] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABC-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:12.200057] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD6-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.209236] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEE-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:56:12.242237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC85-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.255550] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.255558] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B2-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.255576] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B2-0-0] [lt=16][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.255583] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B2-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.276636] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEE-0-0] [lt=25] is primary tenant, no need check readable_scn [2024-12-02 13:56:12.316940] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.355712] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:12.355730] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=268, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:12.355939] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80B-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.360733] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B3-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.361305] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.361315] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.361331] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B3-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.361337] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B3-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.454970] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC67-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:12.457512] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC67-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:12.457533] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC67-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:12.459915] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.466407] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.477266] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.477276] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.477294] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B4-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.477299] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B4-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.485481] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC88-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:12.576737] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.582397] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.583893] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B5-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.583903] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.583920] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.583929] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.680325] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.689141] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.690405] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.690415] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.690430] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.690438] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.784223] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF80F-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.795543] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.797154] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B7-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.797164] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.797180] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.797188] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B7-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.888004] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF810-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.902363] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:12.903705] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B8-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:12.903715] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B8-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:12.903730] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B8-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:12.903739] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:12.991495] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF811-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.008855] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.010457] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.010466] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.010482] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B9-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.010490] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4B9-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.032736] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7A-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:13.047002] INFO [SHARE] update_decay_factor_ (ob_throttle_unit.ipp:284) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99D-0-0] [lt=19] [Throttle] Update Throttle Unit Config(is_adaptive_update=true, N=7.155846548080444336e+01, this=0xffff3e53b208, enable_adaptive_limit_=true, Unit Name=TxShare, Config Specify Resource Limit(MB)=511, Resource Limit(MB)=357, Throttle Trigger(MB)=214, Throttle Percentage=60, Max Duration(us)=7200000000, Decay Factor=1.068302554729592885e+03) [2024-12-02 13:56:13.047172] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99D-0-0] [lt=29] get number(ret=0, service_type_=0, range=1, base_id=1733118973046175000, start_id=1733118973046175000, end_id=1733118973046175001) [2024-12-02 13:56:13.049025] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99D-0-0] [lt=16] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118973046175000, v:0}, tenant_id=1001) [2024-12-02 13:56:13.053060] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC68-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:13.055526] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC68-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:13.056946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC82-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:13.102240] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF812-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.115576] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.117465] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.117478] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BA-0-0] [lt=12][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.117494] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.117503] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.166306] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABD-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:13.168685] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABD-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:13.200131] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:13.207894] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF813-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.222537] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BB-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.224338] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BB-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.224348] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.224365] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.224370] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BB-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.242317] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC86-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:13.281046] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE26-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:56:13.282372] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEF-0-0] [lt=17] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118973136714000, v:0}}], last_dump_ts_=1733118967973755, force_dump=false) [2024-12-02 13:56:13.282397] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEF-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:13.285861] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE27-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:13.288234] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECEF-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:13.288615] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320122][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE28-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:13.288814] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320115][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE29-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:13.289465] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE2A-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:56:13.289658] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE2B-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:56:13.290528] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE31-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:13.291085] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE32-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:56:13.298011] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=10] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:56:13.311734] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF814-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.317014] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD7-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:13.329531] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.331159] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.331170] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.331186] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.331195] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BC-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.355575] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=11] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118972122240000, replayable_scn:1733118972122240000, readable_scn:1733118972122240000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118973352489, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118972122240000, replayable_scn:1733118972122240000, readable_scn:1733118972122240000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118973352489, ora_rowscn_=1733118973285113000, content_changed=true) [2024-12-02 13:56:13.355804] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:13.387492] INFO [RS] prepare (ob_mview_maintenance_task.cpp:178) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=14] mview maintenance task prepare success(tenant_id_=1002, round_=11, prepare_cost_us_=0) [2024-12-02 13:56:13.415909] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF815-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.436288] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.437872] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.437886] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BD-0-0] [lt=12][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.437903] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BD-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.437912] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.440719] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118973352489, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118972122240000, replayable_scn:1733118972122240000, readable_scn:1733118972122240000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:13.460188] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC68-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:13.462745] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC68-0-0] [lt=60] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:13.462768] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC68-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:13.485552] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC89-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:13.519916] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF816-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.542982] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.545689] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.545700] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.545716] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.545725] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.624164] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF817-0-0] [lt=17][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:13.634879] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAE2-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:13.650753] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.653105] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BF-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.653116] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.653133] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.653142] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4BF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.758017] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.767467] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C0-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.767477] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.767491] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C0-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.767497] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C0-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.872413] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C1-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.874023] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.874033] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.874050] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.874059] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:13.887380] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118973885031, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118973885031, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:13.978923] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C2-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:13.980753] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:13.980764] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:13.980779] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:13.980788] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.002946] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE8-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.034161] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7B-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:14.041160] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE4-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.057025] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC83-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.060915] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC69-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:14.063336] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC69-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:14.085680] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.102317] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.102327] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.102343] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.102352] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.163227] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF81C-0-0] [lt=0] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118969111792, force_dump=false) [2024-12-02 13:56:14.175431] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABE-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:14.178946] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABE-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:14.198600] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=8] need remove task(remove_task_ids=[]) [2024-12-02 13:56:14.198619] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=18] remove all timeout ddl task succeed [2024-12-02 13:56:14.200206] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.207204] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C4-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.208746] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.208756] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.208772] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.208781] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.242419] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC87-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.302029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF0-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:14.309642] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF0-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:14.313725] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.316192] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.316202] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.316217] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.316227] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.317095] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD8-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.349340] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC67-0-0] [lt=24] root balance idle(idle_time=10000000) [2024-12-02 13:56:14.349369] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=20] start try disaster recovery [2024-12-02 13:56:14.349382] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:14.350615] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.355250] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:14.355276] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.355291] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=13] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.355350] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=55] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:14.355365] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.355381] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.355418] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=29] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.355438] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.355449] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.355458] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.356060] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:14.357725] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:14.357744] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:14.359016] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.363557] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:14.363588] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=25] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.363600] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.363658] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=54] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:14.363668] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.363683] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.363717] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.363733] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.363739] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=5] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.363749] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.365690] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:14.366992] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:14.368213] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=21] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.372296] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF81E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.372679] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=26] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:14.372708] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=24] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.372720] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.372774] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:14.372787] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.372800] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.372834] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.372850] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.372859] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.372868] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.375164] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.381957] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:14.381981] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.381993] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.382046] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=49] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:14.382055] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.382068] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.382103] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.382118] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.382127] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.382136] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.384029] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:14.384042] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:14.385274] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.389643] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:14.389671] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.389682] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.389735] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=50] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:14.389748] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.389763] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.389797] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.389813] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.389822] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.389831] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.391809] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:14.393043] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:14.394225] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.398574] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:14.398602] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.398609] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.398658] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=45] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:14.398666] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.398676] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=7] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.398706] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=23] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.398716] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=4] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.398722] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=4] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.398727] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.400560] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:14.407283] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:14.407311] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.407323] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:14.407376] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:14.407385] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:14.407401] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:14.407436] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.407452] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.407462] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.407471] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:14.409370] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:14.409389] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=15] start check unit migrate finish [2024-12-02 13:56:14.409404] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:14.415074] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:14.421598] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.425436] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=19] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:14.432619] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C6-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.432626] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.432639] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C6-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.432644] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.435618] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC68-0-0] [lt=22] start to check alter locality finish [2024-12-02 13:56:14.436733] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC69-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.437850] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC69-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.440095] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6B-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.441190] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6B-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.443430] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6D-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.444519] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6D-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.445715] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6D-0-0] [lt=9] start check shrink resource pool [2024-12-02 13:56:14.445739] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6E-0-0] [lt=18] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:14.446792] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6E-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.447891] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6E-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.447972] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6F-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:14.449022] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6F-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.450132] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC6F-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.450211] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC70-0-0] [lt=18] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:14.451260] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC70-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.452353] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC70-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.452433] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC70-0-0] [lt=17] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.452450] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC70-0-0] [lt=16] start check primary zone [2024-12-02 13:56:14.452464] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC71-0-0] [lt=9] start check primary zone(tenant_id=1) [2024-12-02 13:56:14.453508] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC71-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.453582] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC72-0-0] [lt=13] start check primary zone(tenant_id=1002) [2024-12-02 13:56:14.454618] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC72-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.454692] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=14] start check primary zone(tenant_id=1020) [2024-12-02 13:56:14.455723] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:14.455803] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=18] start do unit balance [2024-12-02 13:56:14.458248] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=15] start distribute for server status change [2024-12-02 13:56:14.458301] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=17] finish distribute for server status change(ret=0) [2024-12-02 13:56:14.458312] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=11] finish do unit balance(ret=0) [2024-12-02 13:56:14.460654] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=7] start to do tenant group unit balance [2024-12-02 13:56:14.460696] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=18] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:14.460712] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=11] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:14.460733] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=15] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:14.460760] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=13] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:14.460782] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:14.460795] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:14.460807] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:14.460832] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:14.460872] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=39] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:14.460880] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=7] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:14.460896] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:14.460910] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=13] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:14.465548] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC69-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:14.468142] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC69-0-0] [lt=61] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:14.468164] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC69-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:14.476585] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF81F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.485579] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8A-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:14.537583] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C7-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.539175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C7-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.539185] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.539201] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.539211] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.580319] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF820-0-0] [lt=56] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118969526643, force_dump=false) [2024-12-02 13:56:14.580361] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF820-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.644231] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.646076] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C8-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.646086] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.646102] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.646110] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C8-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.684045] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF821-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.751073] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.752412] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.752422] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.752438] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C9-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.752446] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4C9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.787374] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF822-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.857522] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.871376] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.871387] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CA-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.871403] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.871412] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.890922] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF823-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.976425] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:14.977741] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:14.977748] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CB-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:14.977763] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CB-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:14.977772] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:14.994715] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF824-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.035576] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7C-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:15.057099] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC84-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:15.068744] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6A-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:15.071267] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6A-0-0] [lt=31] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:15.082767] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.084398] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.084409] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.084425] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.084436] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CC-0-0] [lt=10] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.098604] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF825-0-0] [lt=17][errcode=0] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:15.185576] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABF-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:15.187943] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEABF-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:15.189429] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.191149] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CD-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.191155] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CD-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.191171] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CD-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.191180] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.200284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAD9-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:15.242534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC88-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:15.276262] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99D-0-0] [lt=18] get number(ret=0, service_type_=0, range=1, base_id=1733118975275501000, start_id=1733118975275501000, end_id=1733118975275501001) [2024-12-02 13:56:15.277913] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99D-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118975275501000, v:0}, tenant_id=1002) [2024-12-02 13:56:15.296226] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.298685] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.298695] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.298711] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.298720] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.316589] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF1-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:15.317172] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAD9-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:15.323209] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF1-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118975124986001, v:0}}], last_dump_ts_=1733118970008378, force_dump=false) [2024-12-02 13:56:15.323231] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF1-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:15.355757] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:15.355774] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=213, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:15.403689] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.405210] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.405220] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.405235] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.405244] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4CF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.417852] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF828-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.470821] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6A-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:15.473326] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6A-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:15.473346] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6A-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:15.485652] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:15.510216] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.511747] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.511758] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D0-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.511773] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.511781] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.521141] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF829-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.616735] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.618175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.618185] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.618204] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D1-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.618213] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.625109] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.635336] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAE3-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=21, ddl_arg=NULL) [2024-12-02 13:56:15.723278] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.725143] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.725155] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D2-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.725171] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.725180] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.728410] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.830222] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.832061] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.846938] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.846949] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.846962] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D3-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.846967] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D3-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.951945] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D4-0-0] [lt=15] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:15.953480] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D4-0-0] [lt=18][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.953492] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D4-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:15.953508] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:15.953517] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:15.981361] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:15.989728] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:16.007593] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAE9-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.036976] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7D-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:16.041460] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE5-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.057179] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC85-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.058434] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.059738] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.059749] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.059765] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.059773] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D5-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.076655] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6B-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:16.079143] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6B-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:16.085669] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82E-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.133881] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:16.135279] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:16.135338] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:16.135350] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:16.135361] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:16.135369] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:16.135380] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:16.135387] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118981133901, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:16.164851] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.166395] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.166407] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D6-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.166422] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.166432] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D6-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.189169] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF82F-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.194538] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC0-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:16.196894] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC0-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:16.200362] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADA-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.226536] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:16.227705] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976226473, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:16.227732] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981227730, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:16.242642] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC89-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.271221] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D7-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.272468] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.272479] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.272494] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.272502] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.292855] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF830-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.317247] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.321267] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:16.322432] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976321201, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:16.322460] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981322457, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:16.323581] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118976322366, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:16.323606] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981323604, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:16.341666] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF2-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:56:16.355616] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118975124986001, replayable_scn:1733118975124986001, readable_scn:1733118975124986001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118976352148, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118975124986001, replayable_scn:1733118975124986001, readable_scn:1733118975124986001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118976352148, ora_rowscn_=1733118976343723000, content_changed=true) [2024-12-02 13:56:16.355851] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:16.357134] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:16.376016] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF2-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:16.377684] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.379782] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D8-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.379793] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D8-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.379810] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.379820] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.396242] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF831-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.442363] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118976352148, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118975124986001, replayable_scn:1733118975124986001, readable_scn:1733118975124986001, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:16.469771] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE34-0-0] [lt=5] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:16.476107] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6B-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:16.478538] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6B-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:16.478559] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6B-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:16.484964] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.485730] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8C-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:16.486559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D9-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.486569] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.486585] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.486595] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4D9-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.499925] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF832-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.591676] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.593389] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DA-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.593400] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.593416] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.593425] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.603305] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF833-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.698316] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.699600] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.699610] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.699625] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.699634] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.707017] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF834-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.759669] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96C-0-0] [lt=26] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:16.804645] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.806367] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DC-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.806379] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DC-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.806395] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.806404] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.810475] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF835-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.888932] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118976885589, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118976885589, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:16.911336] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:16.912607] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DD-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.912617] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:16.912633] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DD-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:16.912642] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:16.914370] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF836-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:16.978688] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96C-0-0] [lt=30] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:16.979920] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96C-0-0] [lt=19] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:16.981103] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96C-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.017605] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.017729] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF837-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.019150] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.019160] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.019176] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.019184] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.038373] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7E-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:17.057252] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC86-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:17.084302] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6C-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:17.087805] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6C-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:17.103910] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2320160][T1001_BackupDat][T1001][YB42C0A800AF-000628430AAEE96C-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:17.120336] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF838-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.124173] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.124763] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.124774] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.124789] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.124798] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4DF-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.200439] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADB-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:17.203544] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC1-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:17.205938] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC1-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:17.224079] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF839-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.227902] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF839-0-0] [lt=50] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.229752] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.231232] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.231243] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E0-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.231259] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.231268] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.242732] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8A-0-0] [lt=26] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:17.317324] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADB-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:17.328259] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.336248] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.337502] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.337513] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E1-0-0] [lt=10][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:17.337530] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E1-0-0] [lt=1] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.382143] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF3-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:17.386951] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF3-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:17.429796] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=20] start checking lost replicas [2024-12-02 13:56:17.431672] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83B-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.442559] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.443804] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.443814] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.443830] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.443840] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.481204] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:17.483737] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6C-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:17.483758] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6C-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:17.485803] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8D-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:17.535200] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.548769] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.550411] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.550422] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.550438] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E3-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.550447] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.635750] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAE4-0-0] [lt=22] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:17.638512] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.655435] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.656682] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E4-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.656693] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.656709] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.656718] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.741874] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83E-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.761732] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.763227] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.763238] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.763254] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.763263] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E5-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.845242] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.848556] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF83F-0-0] [lt=53] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.868298] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.869559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.869570] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.869586] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.869592] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E6-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:17.922019] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:269) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=15] mview maintenance task gc mview success(tenant_id_=1, round_=11, gc_mview_cost_us_=1086, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:56:17.949002] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF840-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.974604] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:17.976513] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:17.976523] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:17.976539] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:17.976548] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.012457] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAEA-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.039774] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC7F-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:18.041776] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE6-0-0] [lt=21] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.052416] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF841-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.057324] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC87-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.081555] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E8-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.082877] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.082887] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.082903] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.082912] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.093119] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6D-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:18.095594] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6D-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:18.140573] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:56:18.156202] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF842-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.187859] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.189352] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.189362] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.189378] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E9-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.189383] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4E9-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.200516] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADC-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.212741] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC2-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:18.215184] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC2-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:18.242831] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8B-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.259618] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF843-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.294310] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.296308] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EA-0-0] [lt=12][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.296318] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.296335] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.296344] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.317397] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.355971] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:18.363825] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF844-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.392165] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF4-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118978127880002, v:0}}], last_dump_ts_=1733118973282370, force_dump=false) [2024-12-02 13:56:18.392189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF4-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:18.398672] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF4-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:18.401301] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.403011] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EB-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.403022] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.403038] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.403044] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EB-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.467313] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF845-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.485874] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8E-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:18.486456] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:18.489001] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6D-0-0] [lt=44] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:18.489018] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6D-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:18.508063] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.509447] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EC-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.509454] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EC-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.509475] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EC-0-0] [lt=19][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.509480] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EC-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.560025] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE37-0-0] [lt=4] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:18.570649] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF846-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.614454] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4ED-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.615714] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4ED-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.615725] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4ED-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.615741] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4ED-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.615750] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4ED-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.674330] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF847-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.720659] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.721960] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.721970] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EE-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.721986] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.721996] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.778005] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF848-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.826912] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EF-0-0] [lt=28] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.828502] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.828512] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.828528] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.828537] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4EF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.881676] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF849-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.933375] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:18.934639] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:18.934649] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:18.934664] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:18.934672] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:18.985309] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.039475] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.041021] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.041032] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.041049] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.041058] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.041170] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC80-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:19.057394] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC88-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:19.089151] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84B-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.100915] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6E-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:19.103379] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6E-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:19.146210] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.147703] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.147714] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.147730] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.147739] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F2-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.192554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.192811] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84C-0-0] [lt=53] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118974163226, force_dump=false) [2024-12-02 13:56:19.200591] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADD-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:19.221986] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC3-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:19.224371] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC3-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:19.242935] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8C-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:19.252664] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.254411] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F3-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.254422] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.254437] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.254447] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.296265] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84D-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.317475] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADD-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:19.355876] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118977126644002, replayable_scn:1733118977126644002, readable_scn:1733118977126644002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118979352852, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118977126644002, replayable_scn:1733118977126644002, readable_scn:1733118977126644002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118979352852, ora_rowscn_=1733118978394176001, content_changed=true) [2024-12-02 13:56:19.356141] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:19.356157] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=259, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:19.358795] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99E-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733118979358112000, start_id=1733118979358112000, end_id=1733118979358112001) [2024-12-02 13:56:19.359398] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.359499] INFO [LIB] stat (utility.h:1176) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99E-0-0] [lt=21] [PALF STAT PUSH LOG TO FOLLOWERS RPC POST COST TIME](cur_stat_count=11, stat_interval=1000000, avg (count/cost)=0, this=0xffff55fbc130, extra_info="") [2024-12-02 13:56:19.360735] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99E-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118979358112000, v:0}, tenant_id=1) [2024-12-02 13:56:19.360748] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.360758] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F4-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.360774] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F4-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.360784] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.399736] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84E-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.404183] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF5-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:19.409939] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF5-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:56:19.444003] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118979352852, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118977126644002, replayable_scn:1733118977126644002, readable_scn:1733118977126644002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:19.465761] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.467184] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.467195] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F5-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.467211] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F5-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.467220] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.485946] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC8F-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:19.491696] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6E-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:19.494188] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6E-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:19.494206] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6E-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:19.503092] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF84F-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.572106] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.573733] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F6-0-0] [lt=13][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.573744] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.573756] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F6-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.573765] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.606403] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF850-0-0] [lt=52] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118974580316, force_dump=false) [2024-12-02 13:56:19.606436] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF850-0-0] [lt=8][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.636185] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAE5-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:19.678605] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.679911] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.679918] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F7-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.679934] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F7-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.679942] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.710030] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF851-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.784817] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.786329] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.786340] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.786355] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.786364] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.813901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF852-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.890514] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118979887539, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118979887539, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:19.891218] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.892464] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.892475] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.892491] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.892497] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4F9-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:19.917606] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF853-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.997403] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FA-0-0] [lt=22] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:19.998678] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FA-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:19.998688] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:19.998704] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:19.998713] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FA-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.017290] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAEB-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.020937] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF854-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.042080] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE7-0-0] [lt=13] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.042532] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC81-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:20.057477] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC89-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.103581] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FB-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.105331] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.105342] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.105357] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FB-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.105366] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.108679] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6F-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:20.111134] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC6F-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:20.124444] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF855-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.200669] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADE-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.210276] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.211559] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.211570] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.211585] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.211594] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.227782] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF856-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.231193] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC4-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:20.233569] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC4-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:20.243063] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8D-0-0] [lt=53] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.316464] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.317549] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADE-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.331437] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF857-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.335302] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FD-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.335312] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.335328] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FD-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.335337] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.356079] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:20.415360] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF6-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:20.436563] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF858-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.437463] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF6-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118980129546002, v:0}}], last_dump_ts_=1733118975323207, force_dump=false) [2024-12-02 13:56:20.437495] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF6-0-0] [lt=27] is primary tenant, no need check readable_scn [2024-12-02 13:56:20.440100] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.440641] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.440649] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FE-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.440665] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.440674] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.486019] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC90-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:20.496815] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:20.499312] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6F-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:20.499330] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC6F-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:20.540555] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF859-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.545417] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.546702] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FF-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.546708] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FF-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.546724] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.546733] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF4FF-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.644579] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.651475] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF500-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.652974] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF500-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.652985] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF500-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.653001] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF500-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.653010] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF500-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.748855] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.758080] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF501-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.759316] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF501-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.759326] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF501-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.759342] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF501-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.759352] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF501-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.852176] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.864323] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF502-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.865874] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF502-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.865884] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF502-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.865900] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF502-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.865908] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF502-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.960408] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85D-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.970858] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF503-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:20.972068] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF503-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:20.972079] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF503-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:20.972095] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF503-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:20.972100] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF503-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:20.983571] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=7] start do ObArchiveSchedulerService round(round=10) [2024-12-02 13:56:20.987052] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=20][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:56:20.987069] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=16][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1020 AND dest_no = 0 AND name = 'state', exec_tenant_id=1019) [2024-12-02 13:56:20.987137] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:20.987155] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=16][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:56:20.987166] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2324799][T1019_ArchiveSv][T1019][YB42C0A800AF-0006284313EEE955-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1020, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:20.992353] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:20.996351] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=17] failed to get result(ret=-4008) [2024-12-02 13:56:21.043901] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC82-0-0] [lt=13] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:21.057557] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8A-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:21.063597] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.077001] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF504-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.078270] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF504-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.078281] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF504-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.078297] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF504-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.078307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF504-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.116140] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC70-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:21.118519] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC70-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:21.133966] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:21.135360] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:21.135415] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:21.135427] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:21.135439] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:21.135448] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:21.135459] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:21.135466] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118986133980, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:21.167540] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF85F-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.183356] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF505-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.184602] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF505-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.184613] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF505-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.184630] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF505-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.184639] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF505-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.200749] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEADF-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:21.227794] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:21.228928] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981227730, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:21.228955] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986228952, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:21.240152] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC5-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:21.242513] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC5-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:21.243173] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8E-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:21.271329] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF860-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.282412] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE974-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1733118981281455000, start_id=1733118981281455000, end_id=1733118981281455001) [2024-12-02 13:56:21.283828] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE974-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118981281455000, v:0}, tenant_id=1019) [2024-12-02 13:56:21.289505] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF506-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.290931] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF506-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.290942] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF506-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.290959] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF506-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.290967] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF506-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.317624] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEADF-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:21.322522] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:21.323646] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981322457, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:21.323671] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986323669, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:21.324786] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118981323604, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:21.324808] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986324806, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:21.356169] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:21.357450] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:21.374736] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF861-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.395985] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF507-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.397208] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF507-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.397218] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF507-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.397233] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF507-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.397242] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF507-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.443852] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF7-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:56:21.451051] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF7-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:21.478348] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF862-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.486093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC91-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:21.501984] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC70-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:21.502184] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF508-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.503403] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF508-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.503413] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF508-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.503429] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF508-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.503434] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF508-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.504493] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC70-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:21.504513] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC70-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:21.581931] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF863-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.600953] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=19] failed to get result(ret=-4008) [2024-12-02 13:56:21.608426] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF509-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.614478] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF509-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.614488] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF509-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.614500] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF509-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.614505] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF509-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.636626] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAE6-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:21.685397] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF864-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.719418] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50A-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.720747] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50A-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.720757] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50A-0-0] [lt=9][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:21.720774] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50A-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.789297] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF865-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.825680] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.827276] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.827286] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.827301] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.827309] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50B-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.892949] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF866-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.932245] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:21.933492] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:21.933500] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50C-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:21.933514] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:21.933520] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:21.996703] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF867-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.022018] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAEC-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.038474] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.039690] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.039699] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.039714] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.039721] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.042404] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE8-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.045255] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC83-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:22.057634] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8B-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.100013] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF868-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.123739] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC71-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:22.126218] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC71-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:22.144565] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50E-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.153654] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50E-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.153663] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.153675] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50E-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.153683] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50E-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.200829] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE0-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.203687] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF869-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.243284] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC8F-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.249167] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC6-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:22.251557] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC6-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:22.258525] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.259880] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.259890] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.259906] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.259913] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF50F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.307203] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86A-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.317693] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.355966] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118980129546002, replayable_scn:1733118980129546002, readable_scn:1733118980129546002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118982352777, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118980129546002, replayable_scn:1733118980129546002, readable_scn:1733118980129546002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118982352777, ora_rowscn_=1733118981445615001, content_changed=true) [2024-12-02 13:56:22.356200] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:22.364743] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF510-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.365901] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF510-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.365911] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF510-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.365925] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF510-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.365933] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF510-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.410767] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86B-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.445651] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118982352777, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118980129546002, replayable_scn:1733118980129546002, readable_scn:1733118980129546002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:22.456245] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF8-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:22.462039] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF8-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:22.470724] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF511-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.472353] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF511-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.472364] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF511-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.472376] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF511-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.472386] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF511-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.486170] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC92-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:22.507168] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC71-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:22.509656] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC71-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:22.509674] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC71-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:22.514120] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.577174] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF512-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.578390] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF512-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.578400] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF512-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.578416] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF512-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.578425] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF512-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.617750] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.683382] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF513-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.685122] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF513-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.685132] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF513-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.685147] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF513-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.685156] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF513-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.727990] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.790171] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF514-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.791458] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF514-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.791469] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF514-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.791485] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF514-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.791494] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF514-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.831596] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF86F-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.892110] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118982889262, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118982889262, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:22.896384] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF515-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:22.897921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF515-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:22.897932] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF515-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:22.897947] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF515-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:22.897955] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF515-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:22.935153] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF870-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.002904] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF516-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.004190] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF516-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.004200] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF516-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.004215] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF516-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.004221] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF516-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.039436] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF871-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.046616] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC84-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:23.054563] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99E-0-0] [lt=21] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=365, Holding Memory(MB)=61, Tenant Remain Memory(MB)=505, Usable Remain Memory(MB)=303, Last Update Limit Timestamp=1733118983054532, Is Updated=true) [2024-12-02 13:56:23.054739] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99E-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733118983054532000, start_id=1733118983054532000, end_id=1733118983054532001) [2024-12-02 13:56:23.057714] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8C-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:23.061438] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99E-0-0] [lt=18] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118983054532000, v:0}, tenant_id=1001) [2024-12-02 13:56:23.109155] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF517-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.110978] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF517-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.110988] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF517-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.111003] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF517-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.111011] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF517-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.131492] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC72-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:23.133981] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC72-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:23.142985] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF872-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.200905] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE1-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:23.215895] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF518-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.217170] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF518-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.217181] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF518-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.217196] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF518-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.217201] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF518-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.243386] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC90-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:23.245708] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF873-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.258198] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC7-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:23.260569] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC7-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:23.317767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE1-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:23.322064] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF519-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.323613] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF519-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.323621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF519-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.323636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF519-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.323646] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF519-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.349181] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF874-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.356250] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:23.356268] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=231, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:23.388714] INFO [RS] gc_mview (ob_mview_maintenance_task.cpp:269) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=20] mview maintenance task gc mview success(tenant_id_=1002, round_=11, gc_mview_cost_us_=1143, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:56:23.428495] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.429772] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.429783] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.429798] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.429807] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.452910] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF875-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.467398] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF9-0-0] [lt=17] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118983133403003, v:0}}], last_dump_ts_=1733118978392164, force_dump=false) [2024-12-02 13:56:23.467422] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF9-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:23.480229] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECF9-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:56:23.486245] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC93-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:23.512355] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC72-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:23.514908] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC72-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:23.514930] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC72-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:23.534743] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.536555] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.536566] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.536581] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.536586] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.556305] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF876-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.635953] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=20] failed to get result(ret=-4008) [2024-12-02 13:56:23.637074] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAE7-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:56:23.641517] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.642807] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.642818] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.642833] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.642842] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.659739] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF877-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.747778] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.749342] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.749352] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.749368] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.749377] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.763537] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF878-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.854213] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.855419] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.855429] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.855445] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.855453] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.866791] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF879-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.871998] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2324797][T1019_BackupDat][T1019][YB42C0A800AF-0006284313CEE95C-0-0] [lt=20] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:23.960295] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:23.961560] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:23.961570] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:23.961586] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:23.961595] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF51F-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:23.970428] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.026771] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAED-0-0] [lt=23] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.042736] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAE9-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.047980] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC85-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:24.057789] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8D-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.066489] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF520-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.068112] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF520-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.068123] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF520-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.068138] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF520-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.068147] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF520-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.073881] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87B-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.139404] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC73-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:24.141901] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC73-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:24.173169] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF521-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.174536] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF521-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.174547] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF521-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.174563] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF521-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.174572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF521-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.177392] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.200982] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE2-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.243496] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC91-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.267231] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC8-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:24.270647] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC8-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:24.279321] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF522-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.280952] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF522-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.280966] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF522-0-0] [lt=12][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.280981] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF522-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.280987] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF522-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.281036] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87D-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.281309] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87D-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118979192810, force_dump=false) [2024-12-02 13:56:24.317820] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE2-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.356313] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:24.384698] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.385829] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF523-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.387132] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF523-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.387143] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF523-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.387158] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF523-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.387167] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF523-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.460506] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC73-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:56:24.460530] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=15] start try disaster recovery [2024-12-02 13:56:24.460542] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:24.461755] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.466259] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=15] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:24.466285] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.466296] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.466349] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=50] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:24.466359] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.466373] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.466409] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.466424] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.466434] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.466443] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.468426] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:24.468443] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=16] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:24.469739] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.474098] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:24.474125] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.474136] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.474189] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=49] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:24.474201] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.474215] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.474248] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.474264] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.474273] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.474282] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.476099] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:24.477392] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:24.478596] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=19] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.482891] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:24.482918] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.482929] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.482981] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=48] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:24.482991] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.483005] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.483040] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.483056] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.483064] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.483074] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.484947] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.486351] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC94-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.486395] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFA-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:24.487851] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF87F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.491583] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:24.491612] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.491619] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.491675] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=52] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:24.491689] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.491703] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.491741] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.491757] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.491768] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.491778] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.492199] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF524-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.493919] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:24.493940] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:24.494540] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFA-0-0] [lt=24] is primary tenant, no need check readable_scn [2024-12-02 13:56:24.495148] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.499544] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=25] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:24.499573] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.499580] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.499632] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=49] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:24.499645] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.499658] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.499691] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.499707] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.499717] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.499725] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.501647] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:24.501805] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF524-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.501816] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF524-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.501832] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF524-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.501842] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF524-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.502862] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=18] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:24.504034] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=15] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.508397] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:24.508420] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.508431] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.508482] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:24.508495] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.508508] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.508541] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=25] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.508555] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.508565] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.508574] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.510471] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:24.517127] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:24.517152] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.517163] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:24.517214] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:24.517224] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:24.517236] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:24.517271] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.517285] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.517295] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.517300] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:24.517758] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC73-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:24.519311] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:24.519327] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=14] start check unit migrate finish [2024-12-02 13:56:24.519343] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:24.520368] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC73-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:24.520390] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC73-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:24.525320] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:24.535610] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=16] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:24.545801] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC74-0-0] [lt=19] start to check alter locality finish [2024-12-02 13:56:24.546921] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC75-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.548045] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC75-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.550401] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC77-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.551508] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC77-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.553788] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC79-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.554900] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC79-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.556109] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC79-0-0] [lt=15] start check shrink resource pool [2024-12-02 13:56:24.556128] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7A-0-0] [lt=14] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:24.557197] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7A-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.558327] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7A-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.558405] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7B-0-0] [lt=14] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:24.559476] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7B-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.560589] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7B-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.560662] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7C-0-0] [lt=14] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:24.561732] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7C-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.562841] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7C-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.562915] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7C-0-0] [lt=14] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:24.562930] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7C-0-0] [lt=12] start check primary zone [2024-12-02 13:56:24.562943] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7D-0-0] [lt=8] start check primary zone(tenant_id=1) [2024-12-02 13:56:24.564001] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7D-0-0] [lt=7][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.564077] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7E-0-0] [lt=15] start check primary zone(tenant_id=1002) [2024-12-02 13:56:24.565133] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7E-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.565208] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=15] start check primary zone(tenant_id=1020) [2024-12-02 13:56:24.566277] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=11][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:24.566353] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=15] start do unit balance [2024-12-02 13:56:24.567313] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=11] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118984567302}) [2024-12-02 13:56:24.567448] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE38-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:24.568822] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=14] start distribute for server status change [2024-12-02 13:56:24.568873] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=15] finish distribute for server status change(ret=0) [2024-12-02 13:56:24.568883] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=9] finish do unit balance(ret=0) [2024-12-02 13:56:24.571231] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=7] start to do tenant group unit balance [2024-12-02 13:56:24.571277] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=15] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:24.571292] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:24.571310] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=13] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:24.571334] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:24.571354] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=9] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:24.571365] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:24.571376] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:24.571398] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=15] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:24.571437] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=38] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:24.571447] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:24.571462] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:24.571474] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:24.592303] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF880-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.606780] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF525-0-0] [lt=20] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.608049] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF525-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.608060] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF525-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.608077] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF525-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.608086] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF525-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.695803] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF881-0-0] [lt=20] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118979606402, force_dump=false) [2024-12-02 13:56:24.695846] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF881-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.713000] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF526-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.714498] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF526-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.714508] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF526-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.714523] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF526-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.714533] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF526-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.799566] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF882-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.819408] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF527-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.820723] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF527-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.820733] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF527-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.820750] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF527-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.820759] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF527-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:24.903049] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF883-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.925606] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF528-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:24.927206] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF528-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:24.927213] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF528-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:24.927227] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF528-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:24.927234] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF528-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.006772] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF884-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.032094] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF529-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.033621] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF529-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.033632] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF529-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.033647] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF529-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.033653] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF529-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.049319] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC86-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:25.057860] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:25.110507] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF885-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.138496] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.140033] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52A-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.140043] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.140057] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52A-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.140065] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.147291] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC74-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:25.149783] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC74-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:25.201018] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE3-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:25.214257] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF886-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.243623] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC92-0-0] [lt=58] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:25.244927] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.246148] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.246163] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52B-0-0] [lt=14][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.246178] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.246187] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.277227] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC9-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:25.279571] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAC9-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:25.283827] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99E-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733118985283165000, start_id=1733118985283165000, end_id=1733118985283165001) [2024-12-02 13:56:25.285439] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99E-0-0] [lt=22] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118985283165000, v:0}, tenant_id=1002) [2024-12-02 13:56:25.317895] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:25.317999] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF887-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.350987] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52C-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.352675] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.352686] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.352702] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.352711] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.356147] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118983133403003, replayable_scn:1733118983133403003, readable_scn:1733118983133403003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118985352655, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118983133403003, replayable_scn:1733118983133403003, readable_scn:1733118983133403003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118985352655, ora_rowscn_=1733118984489595000, content_changed=true) [2024-12-02 13:56:25.356359] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:25.422158] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF888-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.447174] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118985352655, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118983133403003, replayable_scn:1733118983133403003, readable_scn:1733118983133403003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:25.457734] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.458981] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.458992] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.459008] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52D-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.459018] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.470019] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE3A-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:25.486449] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC95-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:25.501652] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFB-0-0] [lt=16] is primary tenant, no need check readable_scn [2024-12-02 13:56:25.507841] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFB-0-0] [lt=10] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118985285271001, v:0}}], last_dump_ts_=1733118980437462, force_dump=false) [2024-12-02 13:56:25.507867] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFB-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:56:25.523083] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC74-0-0] [lt=32] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:25.525557] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF889-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.525599] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC74-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:25.525615] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC74-0-0] [lt=14] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:25.564012] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.565295] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.565306] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.565322] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52E-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.565332] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.629325] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.637522] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAE8-0-0] [lt=24] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:25.670302] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52F-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.671787] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.671798] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.671813] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.671821] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF52F-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.732957] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.776689] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF530-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.777943] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF530-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.777953] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF530-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.777968] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF530-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.777977] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF530-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.836801] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.882893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF531-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.884370] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF531-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.884377] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF531-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.884391] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF531-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.884399] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF531-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.893765] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118985891337, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118985891337, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:25.940132] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88D-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.989319] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF532-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:25.990672] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF532-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:25.990682] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF532-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:25.990698] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF532-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:25.990706] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF532-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:25.994995] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:26.031481] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAEE-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.043059] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAEA-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.043780] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.050679] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC87-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:26.057937] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC8F-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.095745] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF533-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.134046] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:26.135306] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=3][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:26.135357] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:26.135364] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:26.135372] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:26.135376] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:26.135384] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=5][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:26.135388] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=3][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118991134059, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:26.147206] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF88F-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.154921] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF533-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.154932] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF533-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.154945] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF533-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.154951] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF533-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.155115] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC75-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:26.157613] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC75-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:26.201097] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE4-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.229018] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:26.230218] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986228952, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:26.230247] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=26] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991230244, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:26.243728] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC93-0-0] [lt=40] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.257373] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF890-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.259924] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF534-0-0] [lt=15] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.260741] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF534-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.260751] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF534-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.260767] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF534-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.260777] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF534-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.286305] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACA-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:26.288712] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACA-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:26.317966] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE4-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.323733] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:26.324891] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986323669, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:26.324916] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991324913, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:26.326042] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118986324806, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:26.326064] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991326062, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:26.356456] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:26.356474] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=14] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=253, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:26.357777] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:26.360830] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF891-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.365591] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF535-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.382331] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF535-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.382341] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF535-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.382356] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF535-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.382364] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF535-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.480861] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF892-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.486522] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC96-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:26.487260] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF536-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.488445] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF536-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.488455] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF536-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.488470] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF536-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.488479] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF536-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.513655] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFC-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:56:26.519090] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFC-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:26.528316] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC75-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:26.530878] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC75-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:26.530897] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC75-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:26.584312] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF893-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.593358] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF537-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.594604] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF537-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.594614] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF537-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.594629] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF537-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.594638] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF537-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.688099] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF894-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.699540] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF538-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.700733] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF538-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.700743] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF538-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.700758] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF538-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.700767] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF538-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.759754] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96D-0-0] [lt=23] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:26.791393] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF895-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.805700] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF539-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.807063] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF539-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.807073] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF539-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.807091] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF539-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.807097] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF539-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.895006] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF896-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.912053] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:26.913766] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:26.913777] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:26.913792] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:26.913800] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.985346] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96D-0-0] [lt=32] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:26.986574] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96D-0-0] [lt=18] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:26.987744] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96D-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:26.998451] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF897-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.018788] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.020537] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.020548] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.020563] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.020572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53B-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.052071] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC88-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:27.058015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC90-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:27.102238] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF898-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.125602] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.127737] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.127748] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.127764] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.127769] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53C-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.162976] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC76-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:27.166619] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC76-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:27.201175] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE5-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:27.205780] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF899-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.232761] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.234338] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.234349] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.234364] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.234372] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.243828] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC94-0-0] [lt=35] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:27.295497] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACB-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:27.297925] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACB-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:27.310340] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.313370] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89A-0-0] [lt=55] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.318045] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE5-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:27.339214] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.340624] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53E-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.340636] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.340651] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.340660] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.356537] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:27.413771] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.433520] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=16] start checking lost replicas [2024-12-02 13:56:27.445645] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.447230] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.447241] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.447256] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.447265] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF53F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.486591] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC97-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:27.517819] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.523743] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFD-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:27.529203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFD-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:27.533592] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC76-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:27.536148] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC76-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:27.536167] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC76-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:27.552158] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF540-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.553346] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF540-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.553353] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF540-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.553369] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF540-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.553378] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF540-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.621660] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89D-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.637969] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAE9-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=17, ddl_arg=NULL) [2024-12-02 13:56:27.658359] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF541-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.659720] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF541-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.659730] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF541-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.659745] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF541-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.659754] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF541-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.726483] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89E-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.764637] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF542-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.765876] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF542-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.765886] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF542-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.765902] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF542-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.765911] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF542-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.830050] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF89F-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.870894] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF543-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.872403] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF543-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.872413] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF543-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.872429] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF543-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.872437] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF543-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:27.922100] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=24] mview maintenace task finish(tenant_id_=1, round_=11, status_=100, error_code_=0, cost_us_=20001240, prepare_cost_us_=1, gc_mview_cost_us_=1086, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:56:27.933387] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.936274] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A0-0-0] [lt=53] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.977318] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF544-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:27.978578] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF544-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:27.978585] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF544-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:27.978600] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF544-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:27.978610] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF544-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.036306] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAEF-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.036710] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A1-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.043378] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAEB-0-0] [lt=19] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.053463] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC89-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:28.058093] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC91-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.083624] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF545-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.085152] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF545-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.085163] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF545-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.085179] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF545-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.085188] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF545-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.140306] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.172033] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC77-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:28.174516] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC77-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:28.190290] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF546-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.191708] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF546-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.191718] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF546-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.191734] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF546-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.191740] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF546-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.201250] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.243745] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A3-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.243932] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC95-0-0] [lt=41] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.286334] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE3B-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:28.289066] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE3F-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:28.289292] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320104][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE40-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:56:28.289920] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE41-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-02 13:56:28.290186] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE42-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:56:28.290395] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE43-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:56:28.290884] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320118][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE49-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:28.291546] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE4A-0-0] [lt=12] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:56:28.297334] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF547-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.299203] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF547-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.299215] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF547-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.299233] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF547-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.299243] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF547-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.304960] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACC-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:28.307456] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACC-0-0] [lt=30] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:28.318119] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.348002] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A4-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.356352] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118986135441003, replayable_scn:1733118986135441003, readable_scn:1733118986135441003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118988353486, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118986135441003, replayable_scn:1733118986135441003, readable_scn:1733118986135441003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118988353486, ora_rowscn_=1733118987526338000, content_changed=true) [2024-12-02 13:56:28.356565] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:28.404226] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF548-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.405522] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF548-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.405533] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF548-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.405550] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF548-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.405559] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF548-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.448805] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118988353486, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118986135441003, replayable_scn:1733118986135441003, readable_scn:1733118986135441003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:28.451931] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A5-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.486668] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC98-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:28.510491] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF549-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.512252] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF549-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.512263] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF549-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.512279] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF549-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.512288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF549-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.534645] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFE-0-0] [lt=13] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118988137624003, v:0}}], last_dump_ts_=1733118983467396, force_dump=false) [2024-12-02 13:56:28.534668] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFE-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:28.538817] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC77-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:28.540251] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFE-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:28.541327] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC77-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:28.541345] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC77-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:28.555314] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A6-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.617253] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.619147] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.619158] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.619174] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.619183] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.658641] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.724086] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.725327] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.725338] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.725353] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.725358] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.762596] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A8-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.830267] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54C-0-0] [lt=27] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.831920] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.831930] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.831946] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.831955] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54C-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.871354] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8A9-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.895416] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118988892156, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118988892156, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:28.936923] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:28.938179] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:28.938190] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:28.938206] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:28.938215] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:28.974735] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AA-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.043211] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.045239] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.045249] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.045265] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.045274] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54E-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.054848] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8A-0-0] [lt=29] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:29.058171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC92-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:29.078683] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AB-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.150243] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.151935] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.151942] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54F-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.151958] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.151968] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF54F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.179905] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC78-0-0] [lt=16] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:29.182120] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.182417] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC78-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:29.201330] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE7-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:29.244031] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC96-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:29.256991] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF550-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.258442] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF550-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.258455] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF550-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.258472] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF550-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.258481] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF550-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.313805] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.314078] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AD-0-0] [lt=57] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118984281308, force_dump=false) [2024-12-02 13:56:29.314239] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACD-0-0] [lt=24] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:29.316588] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACD-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:29.318193] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE7-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:29.356609] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:29.356626] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=253, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:29.363361] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF551-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.364657] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF551-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.364669] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF551-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.364685] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF551-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.364694] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF551-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.366249] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99F-0-0] [lt=16] get number(ret=0, service_type_=0, range=1, base_id=1733118989365900000, start_id=1733118989365900000, end_id=1733118989365900001) [2024-12-02 13:56:29.375623] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE99F-0-0] [lt=22] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118989365900000, v:0}, tenant_id=1) [2024-12-02 13:56:29.417391] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AE-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.469693] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF552-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.471295] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF552-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.471306] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF552-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.471322] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF552-0-0] [lt=14][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:29.471332] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF552-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.486740] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC99-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:29.538866] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8AF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.544029] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC78-0-0] [lt=26] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:29.545309] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFF-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:29.546587] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC78-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:29.546605] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC78-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:29.550626] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EECFF-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:56:29.576221] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF553-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.577794] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF553-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.577801] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF553-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.577813] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF553-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.577822] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF553-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.638418] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAEA-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:29.642629] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B0-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.682777] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF554-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.684040] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF554-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.684051] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF554-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.684066] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF554-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.684076] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF554-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.746412] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B1-0-0] [lt=51] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118984695801, force_dump=false) [2024-12-02 13:56:29.746449] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B1-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.788982] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF555-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.790418] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF555-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.790428] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF555-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.790444] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF555-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.790449] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF555-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.850075] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B2-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.895343] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF556-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:29.897809] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF556-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:29.897819] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF556-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:29.897834] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF556-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:29.897843] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF556-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:29.953703] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B3-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.002803] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF557-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.004106] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF557-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.004116] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF557-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.004132] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF557-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.004140] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF557-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.041066] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF0-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.043713] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAEC-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.056218] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8B-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:30.058228] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC93-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.064736] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B4-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.109064] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF558-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.111325] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF558-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.111336] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF558-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.111352] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF558-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.111362] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF558-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.168224] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B5-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.187801] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC79-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:30.190200] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC79-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:30.201409] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE8-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.216267] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF559-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.218171] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF559-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.218182] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF559-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.218198] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF559-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.218207] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF559-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.244135] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC97-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.271467] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.318267] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE8-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.323128] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.323307] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACE-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:30.325718] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACE-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:30.339672] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.339682] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.339699] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.339708] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.356665] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:30.375000] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.444636] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.446039] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55B-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.446053] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55B-0-0] [lt=13][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.446069] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.446078] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.478474] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B8-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.486816] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9A-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:30.549342] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC79-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:30.550979] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.551927] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC79-0-0] [lt=59] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:30.551946] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC79-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:30.552513] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55C-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.552524] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.552539] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.552548] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.555563] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED00-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:30.560873] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED00-0-0] [lt=25] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118990140515003, v:0}}], last_dump_ts_=1733118985507839, force_dump=false) [2024-12-02 13:56:30.560897] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED00-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:56:30.581981] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8B9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.657464] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.658811] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55D-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.658822] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.658838] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.658846] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.685371] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.763683] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.764920] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.764930] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55E-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.764946] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.764954] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.788948] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BB-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.869872] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.871984] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.871994] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.872010] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55F-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.872019] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF55F-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.892532] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BC-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.976824] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF560-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:30.978958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF560-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.978969] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF560-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:30.978984] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF560-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:30.978993] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF560-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:30.996308] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BD-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:30.997805] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:31.057614] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8C-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:31.058306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC94-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:31.083838] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF561-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.094641] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF561-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.094651] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF561-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.094668] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF561-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.094674] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF561-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.099701] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BE-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.134126] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:31.135520] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:31.135580] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:31.135592] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:31.135604] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:31.135612] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:31.135623] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:31.135631] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733118996134145, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:31.195499] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7A-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:31.197956] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7A-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:31.199590] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF562-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.200920] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF562-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.200931] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF562-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.200946] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF562-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.200956] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF562-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.201482] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAE9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:31.203378] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8BF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.230308] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:31.231494] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=17] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991230244, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:31.231520] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996231518, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:31.244237] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC98-0-0] [lt=39] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:31.289119] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE975-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733118991288406000, start_id=1733118991288406000, end_id=1733118991288406001) [2024-12-02 13:56:31.290579] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE975-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118991288406000, v:0}, tenant_id=1019) [2024-12-02 13:56:31.305853] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF563-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.307166] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C0-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.317618] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF563-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.317629] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF563-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.317645] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF563-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.317651] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF563-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.318344] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAE9-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:31.324978] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:31.326154] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991324913, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:31.326180] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996326177, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:31.327310] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118991326062, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:31.327333] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996327332, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:31.332501] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACF-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:31.334928] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEACF-0-0] [lt=15] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:31.356502] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118989138517004, replayable_scn:1733118989138517004, readable_scn:1733118989138517003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118991353628, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118989138517004, replayable_scn:1733118989138517004, readable_scn:1733118989138517003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118991353628, ora_rowscn_=1733118990558490000, content_changed=true) [2024-12-02 13:56:31.356752] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:31.358060] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:31.419344] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C1-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.422601] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF564-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.423924] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF564-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.423934] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF564-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.423950] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF564-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.423959] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF564-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.450453] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118991353628, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118989138517004, replayable_scn:1733118989138517004, readable_scn:1733118989138517003, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:31.486894] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9B-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:31.522946] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.528907] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF565-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.541547] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF565-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.541558] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF565-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.541574] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF565-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.541583] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF565-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.554560] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7A-0-0] [lt=31] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:31.557017] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7A-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:31.557035] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7A-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:31.565857] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED01-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:56:31.571122] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED01-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:56:31.638867] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAEB-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:56:31.640722] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.646512] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF566-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.659442] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF566-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.659452] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF566-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.659467] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF566-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.659475] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF566-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.744339] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C4-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.764447] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF567-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.768415] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF567-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.768425] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF567-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.768441] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF567-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.768449] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF567-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.848643] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C5-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.873451] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF568-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.874974] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF568-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.874984] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF568-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.874999] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF568-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.875005] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF568-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:31.896944] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118991893627, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118991893627, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:31.952048] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C6-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.979870] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF569-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:31.981261] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF569-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:31.981271] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF569-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:31.981285] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF569-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:31.981293] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF569-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.044038] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAED-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.045792] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF1-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.056068] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.058384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC95-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.058971] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8D-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:32.086320] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56A-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.088014] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.088025] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.088041] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.088050] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.159751] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C8-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.193058] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.194526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.194537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.194550] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56B-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.194557] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56B-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.201560] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAEA-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.203161] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7B-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:32.205609] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7B-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:32.244335] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC99-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.263412] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8C9-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.299551] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.301233] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56C-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.301243] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56C-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.301257] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.301266] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.318423] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAEA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.341695] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD0-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:32.344081] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD0-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:32.356766] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:32.356783] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=267, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:32.366927] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.406204] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56D-0-0] [lt=21] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.407452] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.407463] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.407478] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.407484] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56D-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.470344] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CB-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.486972] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9C-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:32.512436] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.514670] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.514680] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.514696] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.514705] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.559688] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7B-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:32.562201] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7B-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:32.562223] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7B-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:32.573809] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CC-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.575879] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED02-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:32.582116] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED02-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:32.619620] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56F-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.620920] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.620930] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.620946] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.620954] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF56F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.677750] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CD-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.725954] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF570-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.727724] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF570-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.727731] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF570-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.727747] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF570-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.727755] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF570-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.794516] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CE-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.832734] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF571-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.834360] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF571-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.834370] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF571-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.834386] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF571-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.834395] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF571-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:32.898199] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8CF-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.939288] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF572-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:32.940529] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF572-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:32.940540] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF572-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:32.940555] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF572-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:32.940564] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF572-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.001820] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D0-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.045484] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF573-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.046727] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF573-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.046738] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF573-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.046754] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF573-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.046763] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF573-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.058462] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC96-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:33.060346] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8E-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:33.066933] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99F-0-0] [lt=19] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=365, Holding Memory(MB)=61, Tenant Remain Memory(MB)=505, Usable Remain Memory(MB)=303, Last Update Limit Timestamp=1733118993066632, Is Updated=true) [2024-12-02 13:56:33.067116] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99F-0-0] [lt=24] get number(ret=0, service_type_=0, range=1, base_id=1733118993066632000, start_id=1733118993066632000, end_id=1733118993066632001) [2024-12-02 13:56:33.069155] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE99F-0-0] [lt=20] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118993066632000, v:0}, tenant_id=1001) [2024-12-02 13:56:33.105145] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.151654] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF574-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.161015] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF574-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.161022] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF574-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.161037] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF574-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.161046] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF574-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.201630] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAEB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:33.209236] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D2-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.210864] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7C-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:33.213293] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7C-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:33.244436] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9A-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:33.265967] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF575-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.267249] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF575-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.267259] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF575-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.267274] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF575-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.267283] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF575-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.312912] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D3-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.318499] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAEB-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:33.350884] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD1-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:33.353286] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD1-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:33.356799] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:33.372098] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF576-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.373810] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF576-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.373820] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF576-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.373836] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF576-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.373845] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF576-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.388793] INFO [RS] finish (ob_mview_maintenance_task.cpp:282) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=20] mview maintenace task finish(tenant_id_=1002, round_=11, status_=100, error_code_=0, cost_us_=20001301, prepare_cost_us_=0, gc_mview_cost_us_=1143, fetch_mview_num_=0, gc_mview_num_=0, gc_stats_num_=0) [2024-12-02 13:56:33.416488] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D4-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.478595] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF577-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.480254] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF577-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.480265] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF577-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.480281] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF577-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.480290] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF577-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.487044] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9D-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:33.519968] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D5-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.564932] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:33.567534] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7C-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:33.567555] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7C-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:33.585096] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF578-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.586442] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF578-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.586452] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF578-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.586468] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF578-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.586477] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF578-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.587842] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED03-0-0] [lt=15] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118993143456004, v:0}}], last_dump_ts_=1733118988534643, force_dump=false) [2024-12-02 13:56:33.587866] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED03-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:33.593189] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED03-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:33.627905] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D6-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.639315] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAEC-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-02 13:56:33.691353] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF579-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.692875] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF579-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.692886] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF579-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.692901] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF579-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.692906] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF579-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.731299] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.797821] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57A-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.799001] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.799010] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.799026] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.799034] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.835070] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.903760] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:33.920258] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:33.920265] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57B-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:33.920282] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57B-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:33.920291] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:33.938316] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8D9-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.025276] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.035886] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.035897] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.035914] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57C-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.035923] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.041777] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DA-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.044336] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAEE-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.050664] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF2-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.058538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC97-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.061722] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC8F-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:34.140836] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.142562] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.142573] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.142589] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57D-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.142598] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.145191] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DB-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.201687] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAEC-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.218595] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7D-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:34.221044] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7D-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:34.244531] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9B-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.247633] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.249219] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.249226] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57E-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.249243] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.249252] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.256800] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DC-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.318538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAEC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.354305] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.355839] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.355850] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.355866] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.355875] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF57F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.356648] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118992141941004, replayable_scn:1733118992141941004, readable_scn:1733118992141941004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118994353448, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118992141941004, replayable_scn:1733118992141941004, readable_scn:1733118992141941004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118994353448, ora_rowscn_=1733118993590659000, content_changed=true) [2024-12-02 13:56:34.356927] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=23] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:34.360040] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD2-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:34.360303] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DD-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.360575] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DD-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118989314076, force_dump=false) [2024-12-02 13:56:34.363547] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD2-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:34.452084] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=16] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118994353448, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118992141941004, replayable_scn:1733118992141941004, readable_scn:1733118992141941004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:34.460808] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF580-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.462274] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF580-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.462285] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF580-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.462300] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF580-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.462308] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF580-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.464129] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DE-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.470274] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE4C-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:34.487118] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.567145] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF581-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.567626] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8DF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.568976] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF581-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.568986] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF581-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.569002] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF581-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.569011] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF581-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.570255] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:34.572055] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC7F-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:56:34.572080] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=16] start try disaster recovery [2024-12-02 13:56:34.572092] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:34.572796] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7D-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:34.572815] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7D-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:34.573316] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.577853] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:34.577878] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.577887] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.577941] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=51] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:34.577951] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.577966] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.578001] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.578017] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.578026] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.578036] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.579994] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:34.580011] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:34.581283] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.585724] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:34.585753] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.585760] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.585816] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=53] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:34.585829] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.585844] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.585879] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.585894] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.585904] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.585913] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.587853] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:34.589093] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:34.590325] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=22] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.594698] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=23] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:34.594724] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.594732] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.594785] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:34.594798] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.594811] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.594845] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.594860] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.594869] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.594878] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.596735] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.598561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED04-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:34.603527] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:34.603552] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=19] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.603563] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.603615] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=49] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:34.603624] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.603637] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.603671] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.603686] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.603695] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.603704] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.604409] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED04-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:56:34.605696] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:34.605711] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:34.606900] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.611223] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:34.611249] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.611256] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.611309] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=50] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:34.611322] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.611336] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.611370] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.611386] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.611395] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.611404] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.613224] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:34.614436] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:34.615618] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=16] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.619957] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:34.619983] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.619994] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.620045] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:34.620058] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.620071] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.620105] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.620120] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.620129] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.620139] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.621964] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:34.628598] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:34.628623] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.628635] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:34.628687] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=49] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:34.628696] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:34.628710] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:34.628744] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.628759] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.628769] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=9] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.628778] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:34.630620] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=4] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:34.630637] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=14] start check unit migrate finish [2024-12-02 13:56:34.630652] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:34.636155] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=8] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:34.646636] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=18] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:34.656786] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC80-0-0] [lt=18] start to check alter locality finish [2024-12-02 13:56:34.657932] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC81-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.659040] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC81-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.661358] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC83-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.662469] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC83-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.664713] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC85-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.665823] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC85-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.667050] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC85-0-0] [lt=16] start check shrink resource pool [2024-12-02 13:56:34.667071] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC86-0-0] [lt=15] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:34.668123] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC86-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.669221] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC86-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.669299] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC87-0-0] [lt=16] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:34.670372] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC87-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.671217] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E0-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.671487] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC87-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.671562] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC88-0-0] [lt=16] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:34.672638] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC88-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.673744] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC88-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.673819] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC88-0-0] [lt=14] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:34.673834] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC88-0-0] [lt=14] start check primary zone [2024-12-02 13:56:34.673847] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC89-0-0] [lt=8] start check primary zone(tenant_id=1) [2024-12-02 13:56:34.674122] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF582-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.674880] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC89-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.674953] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8A-0-0] [lt=14] start check primary zone(tenant_id=1002) [2024-12-02 13:56:34.675977] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8A-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.676049] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=15] start check primary zone(tenant_id=1020) [2024-12-02 13:56:34.677078] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=12][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:34.677152] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=15] start do unit balance [2024-12-02 13:56:34.678120] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=13] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733118994678108}) [2024-12-02 13:56:34.678293] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE4D-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:34.679597] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=15] start distribute for server status change [2024-12-02 13:56:34.679653] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=16] finish distribute for server status change(ret=0) [2024-12-02 13:56:34.679664] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:56:34.681996] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=8] start to do tenant group unit balance [2024-12-02 13:56:34.682036] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=16] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:34.682051] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:34.682071] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=14] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:34.682095] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:34.682117] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:34.682128] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=9] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:34.682137] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=5] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:34.682162] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:34.682203] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=40] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:34.682214] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:34.682229] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:34.682242] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:34.698776] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF582-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.698783] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF582-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.698795] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF582-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.698801] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF582-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.798195] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E1-0-0] [lt=63] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118989746410, force_dump=false) [2024-12-02 13:56:34.798237] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E1-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.803644] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF583-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.805381] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF583-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.805391] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF583-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.805407] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF583-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.805416] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF583-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:34.898519] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118994895459, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118994895459, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:34.901816] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E2-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.910235] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF584-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:34.911543] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF584-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:34.911553] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF584-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:34.911569] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF584-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:34.911575] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF584-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.005238] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.016398] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF585-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.017723] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF585-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.017733] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF585-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.017749] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF585-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.017757] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF585-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.058614] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC98-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:35.063071] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC90-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:35.108864] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E4-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.122665] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF586-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.123877] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF586-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.123886] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF586-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.123901] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF586-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.123910] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF586-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.201765] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAED-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:35.212250] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E5-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.226404] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7E-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:35.228675] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF587-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.228749] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7E-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:35.244633] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9C-0-0] [lt=37] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:35.248193] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF587-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.248203] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF587-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.248218] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF587-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.248223] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF587-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.290981] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99F-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1733118995290901000, start_id=1733118995290901000, end_id=1733118995290901001) [2024-12-02 13:56:35.292563] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE99F-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118995290901000, v:0}, tenant_id=1002) [2024-12-02 13:56:35.316140] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E6-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.318615] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAED-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:35.353157] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF588-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.354335] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF588-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.354346] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF588-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.354361] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF588-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.354368] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF588-0-0] [lt=6] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.356899] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:35.370103] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD3-0-0] [lt=26] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:35.372459] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD3-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:35.435554] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.459192] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF589-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.460430] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF589-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.460439] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF589-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.460454] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF589-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.460462] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF589-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.487193] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEEC9F-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:35.539272] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E8-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.565336] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.566848] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.566859] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.566874] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.566883] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.575492] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7E-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:35.578062] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7E-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:35.578081] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7E-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:35.610099] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED05-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:35.616221] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED05-0-0] [lt=21] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118995291955002, v:0}}], last_dump_ts_=1733118990560870, force_dump=false) [2024-12-02 13:56:35.616246] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED05-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:56:35.639712] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAED-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:35.642592] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8E9-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.671835] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.673116] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.673126] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.673139] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58B-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.673145] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.745950] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8EA-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.778027] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.790079] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.790089] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.790104] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.790113] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58C-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.849808] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8EB-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.894998] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:35.896511] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:35.896521] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:35.896537] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58D-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:35.896546] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:35.953458] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8EC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.000398] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:36.001484] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.014744] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.014754] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.014770] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.014779] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.044652] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAEF-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.055507] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF3-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.056864] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8ED-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.058690] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC99-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.064430] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC91-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:36.119695] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.120916] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.120927] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.120939] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58F-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.120944] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF58F-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.134211] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:36.135590] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=3][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:36.135645] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:36.135656] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:36.135667] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:36.135674] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:36.135686] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:36.135694] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733119001134229, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:36.160656] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8EE-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.201844] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAEE-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.225895] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF590-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.229073] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF590-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.229084] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF590-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.229099] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF590-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.229108] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF590-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.231586] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:36.232775] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996231518, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:36.232803] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=25] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001232800, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:36.234007] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7F-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:36.236419] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC7F-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:36.244731] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9D-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.264090] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8EF-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.318690] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAEE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.326242] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:36.327408] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996326177, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:36.327426] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001327423, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:36.328561] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733118996327332, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:36.328576] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=13] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001328574, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:36.334068] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF591-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.356959] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:36.356975] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=263, return_code_array=[0], broadcast_times=2) [2024-12-02 13:56:36.358261] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:36.367803] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F0-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.379042] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD4-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:36.381408] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD4-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:36.408649] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF591-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.408660] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF591-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.408673] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF591-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.408678] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF591-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.487239] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA0-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:36.513557] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF592-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.514815] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF592-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.514826] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF592-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.514841] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF592-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.514850] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF592-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.529395] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F1-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.580794] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7F-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:36.583358] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7F-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:36.583378] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC7F-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:36.619683] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF593-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.621857] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF593-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.621868] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF593-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.621884] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF593-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.621892] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF593-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.622004] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED06-0-0] [lt=4] is primary tenant, no need check readable_scn [2024-12-02 13:56:36.628932] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED06-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:36.632930] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.726893] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF594-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.728362] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF594-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.728374] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF594-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.728390] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF594-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.728400] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF594-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.736396] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F3-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.759838] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96E-0-0] [lt=24] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:36.833309] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF595-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.834609] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF595-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.834620] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF595-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.834635] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF595-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.834641] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF595-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.840165] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F4-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.939548] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF596-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:36.940885] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF596-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.940895] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF596-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:36.940911] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF596-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:36.940920] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF596-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:36.943783] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F5-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:36.992002] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96E-0-0] [lt=32] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:36.993218] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96E-0-0] [lt=17] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:36.994413] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96E-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.045816] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF597-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.047164] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.047414] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF597-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.047424] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF597-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.047443] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF597-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.047452] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF597-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.058767] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9A-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:37.065813] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC92-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:37.149875] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F7-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.152524] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF598-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.153895] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF598-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.153906] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF598-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.153922] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF598-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.153932] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF598-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.201923] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAEF-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:37.241799] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC80-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:37.244826] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9E-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:37.245332] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC80-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:37.252831] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.258899] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF599-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.260397] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF599-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.260408] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF599-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.260424] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF599-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.260429] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF599-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.318764] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAEF-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:37.356708] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F9-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.356862] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118995291955002, replayable_scn:1733118995291955002, readable_scn:1733118995291955002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733118997353655, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118995291955002, replayable_scn:1733118995291955002, readable_scn:1733118995291955002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733118997353655, ora_rowscn_=1733118996624305000, content_changed=true) [2024-12-02 13:56:37.357089] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:37.365390] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.367627] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8F9-0-0] [lt=57] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.368689] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.368700] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.368716] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.368725] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.388014] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD5-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:37.390394] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD5-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:37.437370] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=14] start checking lost replicas [2024-12-02 13:56:37.453698] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733118997353655, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118995291955002, replayable_scn:1733118995291955002, readable_scn:1733118995291955002, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:37.468006] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FA-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.473761] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.475330] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.475341] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.475357] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.475366] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.487312] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA1-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:37.571847] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FB-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.580278] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59C-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.581526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.581537] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.581552] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.581562] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59C-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.586061] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC80-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:37.588573] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC80-0-0] [lt=47] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:37.588591] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC80-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:37.634940] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED07-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:37.640132] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAEE-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:37.640969] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED07-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:37.675194] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FC-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.686482] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.687696] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.687706] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.687722] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59D-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.687731] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.778512] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FD-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.792654] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.794288] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.794299] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.794314] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.794322] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.881950] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FE-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.899255] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:37.900191] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733118997897798, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733118997897798, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:37.900492] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59F-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.900502] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:37.900518] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:37.900524] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF59F-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:37.943862] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2319901][T1_TntSharedTim][T1][Y0-0000000000000000-0-0] [lt=12] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:56:37.973690] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2319871][T1_BackupDataSr][T1][YB42C0A800AF-0006284308DEE969-0-0] [lt=19] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:37.985281] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:37.988672] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF8FF-0-0] [lt=51] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.005328] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.006872] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A0-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.006883] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.006900] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A0-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.006909] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.044971] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF0-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.058843] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9B-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.060409] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF4-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.067172] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC93-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:38.089116] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF900-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.111873] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.113267] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.113278] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.113294] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.113303] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.142005] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=21] failed to get result(ret=-4008) [2024-12-02 13:56:38.192566] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF901-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.202000] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF0-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.218306] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.219804] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.219815] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.219831] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A2-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.219840] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A2-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.239858] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE50-0-0] [lt=6] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:56:38.244928] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEEC9F-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.250808] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC81-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:38.253280] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC81-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:38.296385] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF902-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.318836] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF0-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.324823] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A3-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.326200] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.326211] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A3-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.326227] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.326236] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A3-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.357105] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=8] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:38.397091] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD6-0-0] [lt=17] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:38.399502] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD6-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:38.399852] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF903-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.431136] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A4-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.432638] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.432650] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.432665] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.432671] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A4-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.487388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA2-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:38.503538] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF904-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.537647] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.539764] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A5-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.539775] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.539791] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.539799] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.591284] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC81-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:38.593858] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC81-0-0] [lt=58] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:38.593879] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC81-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:38.607161] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF905-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.644715] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.646370] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A6-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.646376] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A6-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.646392] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.646401] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A6-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.647200] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED08-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733118998148262004, v:0}}], last_dump_ts_=1733118993587840, force_dump=false) [2024-12-02 13:56:38.647222] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED08-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:38.663229] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED08-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:38.712321] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF906-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.751406] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A7-0-0] [lt=21] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.753573] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.753584] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.753601] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A7-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.753610] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.815808] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF907-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.858634] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A8-0-0] [lt=31] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.869061] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.869068] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A8-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.869081] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A8-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.869090] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:38.919523] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF908-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.973996] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A9-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:38.975526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A9-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:38.975533] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A9-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:38.975549] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:38.975559] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5A9-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.032265] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF909-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.058919] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9C-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:39.068555] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC94-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:39.080460] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.081816] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.081823] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AA-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.081839] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AA-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.081848] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.135663] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.186718] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.188138] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.188148] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.188164] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.188173] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AB-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.202077] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF1-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:39.239334] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.245039] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA0-0-0] [lt=45] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:39.258593] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC82-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:39.261070] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC82-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:39.293097] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.294405] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AC-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.294415] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.294431] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.294440] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.318906] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF1-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:39.343337] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90C-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.357128] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:39.382028] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE9A0-0-0] [lt=23] get number(ret=0, service_type_=0, range=1, base_id=1733118999381509000, start_id=1733118999381509000, end_id=1733118999381509001) [2024-12-02 13:56:39.384282] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE9A0-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733118999381509000, v:0}, tenant_id=1) [2024-12-02 13:56:39.399479] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.400777] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AD-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.400788] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.400805] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AD-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.400815] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AD-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.406328] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD7-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:39.408754] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD7-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:39.447437] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90D-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.447703] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90D-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118994360574, force_dump=false) [2024-12-02 13:56:39.487463] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA3-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:39.505637] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AE-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.506931] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.506939] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AE-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.506955] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.506964] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.550972] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.596426] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC82-0-0] [lt=27] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:39.598868] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC82-0-0] [lt=49] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:39.598887] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC82-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:39.611811] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.613462] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AF-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.613482] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AF-0-0] [lt=19][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.613498] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.613508] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5AF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.640552] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAEF-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:39.654291] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF90F-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.668169] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED09-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:39.675131] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED09-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:39.718326] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B0-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.719983] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.719993] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.720010] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B0-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.720015] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B0-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.757619] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF910-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.824895] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.826154] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.826165] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.826177] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B1-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.826186] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.861251] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF911-0-0] [lt=51] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118994798193, force_dump=false) [2024-12-02 13:56:39.861284] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF911-0-0] [lt=10][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.931023] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:39.932273] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:39.932283] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B2-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:39.932298] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B2-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:39.932307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:39.964478] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF912-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.037149] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.038792] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.038802] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.038817] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.038825] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B3-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.045306] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF1-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.058995] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9D-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.065269] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF5-0-0] [lt=21] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.067941] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF913-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.069973] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC95-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:40.143732] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.145274] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.145285] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.145301] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.145307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B4-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.171321] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF914-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.202157] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF2-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.245129] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA1-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.250155] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.252068] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.252078] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.252094] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.252104] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.266432] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC83-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:40.268866] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC83-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:40.274987] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF915-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.318981] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF2-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.356994] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118998649224000, replayable_scn:1733118998649224000, readable_scn:1733118998148262004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119000354032, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118998649224000, replayable_scn:1733118998649224000, readable_scn:1733118998148262004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119000354032, ora_rowscn_=1733118999670992000, content_changed=true) [2024-12-02 13:56:40.357125] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.357254] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=18] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:40.357268] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=255, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:40.366472] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.366483] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.366499] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.366505] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B6-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.378405] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF916-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.415379] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD8-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:40.417776] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD8-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:40.455353] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=13] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119000354032, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733118998649224000, replayable_scn:1733118998649224000, readable_scn:1733118998148262004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:40.471497] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.472797] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.472805] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B7-0-0] [lt=6][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:40.472823] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B7-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.482042] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF917-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.487536] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA4-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:40.558498] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97E-0-0] [lt=15] [RESTORE] restore scheduler quit [2024-12-02 13:56:40.559704] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97E-0-0] [lt=16] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_recover_table_job) [2024-12-02 13:56:40.560836] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE97F-0-0] [lt=21] get targets(ret=0, jobs=[], exec_tenant_id=1001, sql=select * from __all_import_table_job) [2024-12-02 13:56:40.560883] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE980-0-0] [lt=20][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733119000560858, ref_timestamp=1733118940556826, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:56:40.560970] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE981-0-0] [lt=14] [RESTORE] restore scheduler start [2024-12-02 13:56:40.560980] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE982-0-0] [lt=9] [RESTORE] try process restore job [2024-12-02 13:56:40.562066] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2320157][T1001_REST_SER][T1001][YB42C0A800AF-000628430A8EE982-0-0] [lt=3] [RESTORE] get restore jobs(ret=0, job_cnt=0) [2024-12-02 13:56:40.577787] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B8-0-0] [lt=21] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.579412] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B8-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.579423] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B8-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.579439] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.579445] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B8-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.585418] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF918-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.601450] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC83-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:40.603922] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC83-0-0] [lt=67] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:40.603945] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC83-0-0] [lt=23] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:40.679735] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0A-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:56:40.684387] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.685670] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B9-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.685681] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.685697] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.685703] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5B9-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.685802] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0A-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119000650773000, v:0}}], last_dump_ts_=1733118995616219, force_dump=false) [2024-12-02 13:56:40.685825] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0A-0-0] [lt=19] is primary tenant, no need check readable_scn [2024-12-02 13:56:40.689029] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF919-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.790676] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.792336] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.792387] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.792396] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BA-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.792413] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BA-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.792422] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.895897] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.897327] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:40.898595] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BB-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:40.898606] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:40.898622] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:40.898628] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BB-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:40.901755] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119000899159, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119000899159, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:40.949071] INFO [RS] do_work (ob_tenant_snapshot_scheduler.cpp:123) [2324817][T1019_SnapSche][T1019][YB42C0A800AF-00062843140EE953-0-0] [lt=20] tenant status is not valid(user_tenant_id=1020, status_satisfied=false) [2024-12-02 13:56:40.997772] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2324878][T1019_DBMSSched][T1019][Y0-0000000000000000-0-0] [lt=20] failed to get result(ret=-4008) [2024-12-02 13:56:40.999332] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.003185] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:41.003683] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.024524] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BC-0-0] [lt=13][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.024535] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.024548] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BC-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.024554] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BC-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.059028] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9E-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:41.071363] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC96-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:41.103467] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91D-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.128301] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=28] [RESTORE] restore scheduler quit [2024-12-02 13:56:41.129502] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96A-0-0] [lt=17] get targets(ret=0, jobs=[], exec_tenant_id=1019, sql=select * from __all_recover_table_job) [2024-12-02 13:56:41.129651] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BD-0-0] [lt=15] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.130625] INFO [SHARE] get_all_import_table_jobs (ob_import_table_persist_helper.cpp:109) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96B-0-0] [lt=21] get targets(ret=0, jobs=[], exec_tenant_id=1019, sql=select * from __all_import_table_job) [2024-12-02 13:56:41.130672] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96C-0-0] [lt=20][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733119001130646, ref_timestamp=1733118941125037, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:56:41.130761] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96D-0-0] [lt=15] [RESTORE] restore scheduler start [2024-12-02 13:56:41.130771] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96E-0-0] [lt=10] [RESTORE] try process restore job [2024-12-02 13:56:41.131359] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BD-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.131370] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.131386] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.131393] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BD-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.132027] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96E-0-0] [lt=4] [RESTORE] get restore jobs(ret=0, job_cnt=1) [2024-12-02 13:56:41.133346] INFO [RS.RESTORE] check_all_ls_restore_to_consistent_scn_finish_ (ob_restore_scheduler.cpp:1402) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96E-0-0] [lt=14] check all ls restore to consistent_scn not finish, just wait(ret=0, ret="OB_SUCCESS", tenant_id=1020, tenant_restore_status={status:0}) [2024-12-02 13:56:41.133367] INFO [RS.RESTORE] process_restore_job (ob_restore_scheduler.cpp:216) [2324794][T1019_REST_SER][T1019][YB42C0A800AF-0006284313AEE96E-0-0] [lt=20] [RESTORE] doing restore(ret=0, job={restore_key:{tenant_id:1020, job_id:11}, initiator_job_id:11, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:3, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:56:41.134294] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:41.135680] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:41.135738] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:41.135749] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:41.135761] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:41.135770] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:41.135781] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:41.135788] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733119006134312, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:41.202235] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF3-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:41.207791] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.232864] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:41.234046] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001232800, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:41.234072] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=23] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006234069, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:41.236333] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.238185] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.238195] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.238211] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.238220] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.245223] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA2-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:41.274236] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC84-0-0] [lt=11] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:41.276689] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC84-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:41.295134] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE976-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733119001294678000, start_id=1733119001294678000, end_id=1733119001294678001) [2024-12-02 13:56:41.296865] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE976-0-0] [lt=22] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119001294678000, v:0}, tenant_id=1019) [2024-12-02 13:56:41.312198] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF91F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.319059] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF3-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:41.327487] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:41.328652] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001327423, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:41.328681] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=22] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006328675, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:41.329827] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119001328574, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:41.329849] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006329847, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:41.343106] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.344448] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BF-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.344458] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.344474] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.344482] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5BF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.357166] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:41.358451] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:41.415609] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF920-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.424418] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD9-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:41.426781] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAD9-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:41.449302] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.451282] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.451292] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.451308] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.451316] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.487612] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA5-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:41.526210] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF921-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.556093] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.558136] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C1-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.558146] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.558163] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.558172] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.602401] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320250][T1001_DBMSSched][T1001][Y0-0000000000000000-0-0] [lt=19] failed to get result(ret=-4008) [2024-12-02 13:56:41.606656] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC84-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:41.609225] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC84-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:41.609246] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC84-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:41.629603] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF922-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.640976] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAF0-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:41.663294] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C2-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.674761] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C2-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.674768] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C2-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.674782] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C2-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.674787] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C2-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.690845] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0B-0-0] [lt=9] is primary tenant, no need check readable_scn [2024-12-02 13:56:41.697203] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0B-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:41.733353] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF923-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.779635] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C3-0-0] [lt=13] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.780840] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C3-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.780848] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C3-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.780863] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C3-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.780872] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C3-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.838028] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF924-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.885729] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.887745] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.887755] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C4-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.887771] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C4-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.887780] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:41.941379] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF925-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.992594] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C5-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:41.993894] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:41.993905] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C5-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:41.993917] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C5-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:41.993926] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.045657] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF2-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.046145] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF926-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.059106] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EEC9F-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.070160] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF6-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.072732] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC97-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:42.098804] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.100298] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C6-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.100308] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.100324] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.100333] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.150072] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF927-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.202313] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF4-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.205205] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.206456] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.206467] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C7-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.206483] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.206492] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.245316] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA3-0-0] [lt=31] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.253888] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF928-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.281954] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC85-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:42.284366] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC85-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:42.311292] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C8-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.312604] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.312615] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.312632] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.312641] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C8-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.319132] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF4-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.357247] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:42.373867] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF929-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.417554] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.418990] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.419001] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.419018] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C9-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.419027] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5C9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.433424] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADA-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:42.435818] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADA-0-0] [lt=28] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:42.477473] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.487689] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA6-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:42.523827] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.539480] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.539491] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CA-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.539507] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.539516] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.580803] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.611947] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC85-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:42.614513] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC85-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:42.614532] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC85-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:42.644359] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CB-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.645986] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.645996] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.646012] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.646021] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.684469] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.703786] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0C-0-0] [lt=14] is primary tenant, no need check readable_scn [2024-12-02 13:56:42.709068] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0C-0-0] [lt=30] is primary tenant, no need check readable_scn [2024-12-02 13:56:42.750874] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.752293] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CC-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.752304] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.752320] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.752325] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CC-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.787849] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.857279] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.858777] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CD-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.858788] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.858803] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.858812] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CD-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.892355] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92E-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.963693] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:42.969271] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CE-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:42.969281] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:42.969297] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:42.969306] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CE-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:42.995746] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF92F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.059180] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:43.073955] INFO [SHARE] adaptive_update_limit (ob_share_throttle_define.cpp:112) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE9A0-0-0] [lt=21] adaptive update(Tenant ID=1001, Config Specify Resource Limit(MB)=511, TxShare Current Memory Limit(MB)=365, Holding Memory(MB)=61, Tenant Remain Memory(MB)=505, Usable Remain Memory(MB)=303, Last Update Limit Timestamp=1733119003073399, Is Updated=true) [2024-12-02 13:56:43.074119] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC98-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:43.074144] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE9A0-0-0] [lt=25] get number(ret=0, service_type_=0, range=1, base_id=1733119003073399000, start_id=1733119003073399000, end_id=1733119003073399001) [2024-12-02 13:56:43.074220] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.075767] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.075778] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.075794] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.075802] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5CF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.076036] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE9A0-0-0] [lt=19] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119003073399000, v:0}, tenant_id=1001) [2024-12-02 13:56:43.099018] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF930-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.180710] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D0-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.181945] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D0-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.181952] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D0-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.181968] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.181977] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.202388] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF5-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:43.203045] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF931-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.245410] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA4-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:43.286214] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320101][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE52-0-0] [lt=5] [DDL] execute ddl like stmt(ret=0, cost=12, ddl_arg=NULL) [2024-12-02 13:56:43.286506] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE53-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:56:43.287114] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D1-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.288521] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.288532] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D1-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.288552] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D1-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.288561] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.289989] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC86-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:43.291059] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320103][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE58-0-0] [lt=23] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:43.291248] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320102][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE59-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:56:43.292558] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC86-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:43.292717] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE5A-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:43.292912] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE5B-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=4, ddl_arg=NULL) [2024-12-02 13:56:43.293504] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320107][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE5C-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:43.293702] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE5D-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:43.293865] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE5E-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:56:43.294736] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320100][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE61-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:56:43.295214] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE62-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:56:43.295471] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320106][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE63-0-0] [lt=13] [DDL] execute ddl like stmt(ret=0, cost=2, ddl_arg=NULL) [2024-12-02 13:56:43.299632] INFO [RS] runTimerTask (ob_collect_mv_merge_info_task.cpp:452) [2320369][T1002_TntShared][T1002][Y0-0000000000000000-0-0] [lt=16] collect_mv_merge_info task(ret=0, ret="OB_SUCCESS", need_schedule=false, latest_merge_scn={val:18446744073709551615, v:3}, major_mv_merge_scn={val:18446744073709551615, v:3}) [2024-12-02 13:56:43.306543] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF932-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.319212] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF5-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:43.357011] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119001651462000, replayable_scn:1733119001651462000, readable_scn:1733119001651462000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119003354485, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119001651462000, replayable_scn:1733119001651462000, readable_scn:1733119001651462000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119003354485, ora_rowscn_=1733119002706203000, content_changed=true) [2024-12-02 13:56:43.357240] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:43.393582] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D2-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.395298] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D2-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.395309] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D2-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.395326] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D2-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.395335] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D2-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.410239] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF933-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.442295] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADB-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:43.444604] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADB-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:43.456986] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119003354485, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119001651462000, replayable_scn:1733119001651462000, readable_scn:1733119001651462000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:43.470470] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE65-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:43.487763] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA7-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:43.500283] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D3-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.501905] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D3-0-0] [lt=11][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.501917] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D3-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.501934] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D3-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.501943] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D3-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.517999] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF934-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.606886] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D4-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.608565] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D4-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.608576] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.608591] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.608599] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D4-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.617163] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC86-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:43.619691] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC86-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:43.619709] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC86-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:43.621275] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF935-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.637264] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320439][T1002_DBMSSched][T1002][Y0-0000000000000000-0-0] [lt=21] failed to get result(ret=-4008) [2024-12-02 13:56:43.641426] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAF1-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:56:43.713590] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.714839] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.714846] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.714862] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.714871] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D5-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.717838] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0D-0-0] [lt=12] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119003153404004, v:0}}], last_dump_ts_=1733118998647198, force_dump=false) [2024-12-02 13:56:43.717860] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0D-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:43.723854] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0D-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:43.725452] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF936-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.819734] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D6-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.821005] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D6-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.821015] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.821031] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.821040] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.829092] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF937-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.903181] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119003900815, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119003900815, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:43.925932] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D7-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:43.927597] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D7-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:43.927608] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:43.927623] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:43.927632] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D7-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:43.932492] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF938-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.032495] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.036193] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF939-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.042412] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.042419] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D8-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.042435] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.042440] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D8-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.045909] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF3-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.059256] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA1-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.075045] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF7-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.075431] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC99-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:44.143748] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.147399] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D9-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.147946] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.147956] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.147972] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.147980] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5D9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.198775] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2588) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=4] need remove task(remove_task_ids=[]) [2024-12-02 13:56:44.198795] INFO [RS] remove_inactive_ddl_task (ob_ddl_scheduler.cpp:2607) [2320739][HeartBeatCheckT][T0][Y0-0000000000000000-0-0] [lt=18] remove all timeout ddl task succeed [2024-12-02 13:56:44.202463] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF6-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.245502] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA5-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.247515] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93B-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.252880] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.254139] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DA-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.254150] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DA-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.254166] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.254175] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.297967] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC87-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:44.300446] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC87-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:44.319289] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF6-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.351426] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.357397] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:44.357413] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=262, return_code_array=[0], broadcast_times=2) [2024-12-02 13:56:44.359123] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DB-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.361039] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.361051] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DB-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.361067] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DB-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.361076] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.412756] INFO [RS] recover_task (ob_ddl_scheduler.cpp:2516) [2320738][DDLScanTask][T0][Y0-0000000000000000-0-0] [lt=18] start processing ddl recovery(ddl_event_info={addr_:"192.168.0.175:2882", event_ts_:1733119004412754, sub_id_:0, trace_id_:Y0-0000000000000000-0-0, parent_trace_id_:Y0-0000000000000000-0-0}, task_records=[], primary_tenant_ids=[1002]) [2024-12-02 13:56:44.451321] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADC-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:44.452424] INFO [RS] try_update_primary_ip_list (ob_common_ls_service.cpp:368) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADC-0-0] [lt=28] log restore source not exists or the log restore source type is not service(item={tenant_id:1020, id:1, until_scn:{val:1732874528445031000, v:0}, type:2, value:"file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"}) [2024-12-02 13:56:44.454759] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADC-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:44.456946] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.457229] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93D-0-0] [lt=57] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733118999447702, force_dump=false) [2024-12-02 13:56:44.466017] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DC-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.487817] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA8-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.491613] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DC-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.491624] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DC-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.491639] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DC-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.491648] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.590286] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.596577] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DD-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.598084] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DD-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.598095] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DD-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.598109] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DD-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.598120] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DD-0-0] [lt=10] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.622445] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC87-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:44.624988] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC87-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:44.625008] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC87-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:44.682835] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8B-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:56:44.682862] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=19] start try disaster recovery [2024-12-02 13:56:44.682876] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=11] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:44.684050] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.688550] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:44.688579] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.688587] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.688641] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=51] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:44.688651] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.688666] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=14] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.688702] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=28] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.688716] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.688726] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.688735] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.690673] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:44.690694] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=19] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:44.692021] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.694062] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF93F-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.696383] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=24] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:44.696412] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.696423] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.696480] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=53] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:44.696490] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.696505] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.696541] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.696558] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.696568] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.696578] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.698641] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=5] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:44.699925] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:44.701171] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.703215] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.704776] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DE-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.704787] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.704803] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.704811] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DE-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.705539] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:44.705563] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=20] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.705571] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.705624] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:44.705637] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.705650] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.705684] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.705699] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.705709] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.705718] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.707567] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.714287] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=19] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:44.714313] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.714324] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.714378] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=50] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:44.714388] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.714401] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.714436] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.714451] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.714460] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.714469] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.716260] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:44.716276] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:44.717464] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=5] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.721742] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=31] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:44.721767] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.721778] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.721830] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=48] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:44.721843] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.721856] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.721890] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.721906] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.721915] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.721924] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.723657] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:44.724838] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:44.726020] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=18] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.729278] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0E-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:44.730309] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:44.730337] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.730349] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.730402] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:44.730415] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.730428] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.730462] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.730478] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.730488] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.730497] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.732442] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:44.734862] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0E-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:56:44.739139] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:44.739167] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.739179] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:44.739231] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=49] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:44.739240] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:44.739253] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=12] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:44.739288] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.739304] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.739313] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.739322] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=8] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:44.741122] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=5] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:44.741139] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=14] start check unit migrate finish [2024-12-02 13:56:44.741157] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=10] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:44.746717] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=9] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:44.756884] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=19] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:44.767095] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8C-0-0] [lt=22] start to check alter locality finish [2024-12-02 13:56:44.768260] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8D-0-0] [lt=22][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.769398] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8D-0-0] [lt=21][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.771724] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8F-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.772836] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC8F-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.775153] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC91-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.776257] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC91-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.777466] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC91-0-0] [lt=17] start check shrink resource pool [2024-12-02 13:56:44.777498] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC92-0-0] [lt=26] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:44.778559] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC92-0-0] [lt=10][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.779676] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC92-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.779754] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC93-0-0] [lt=18] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:44.780814] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC93-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.781936] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC93-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.782012] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC94-0-0] [lt=18] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:44.783071] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC94-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.784173] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC94-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.784250] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC94-0-0] [lt=18] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:44.784268] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC94-0-0] [lt=16] start check primary zone [2024-12-02 13:56:44.784282] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC95-0-0] [lt=9] start check primary zone(tenant_id=1) [2024-12-02 13:56:44.785326] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC95-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.785401] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC96-0-0] [lt=17] start check primary zone(tenant_id=1002) [2024-12-02 13:56:44.786458] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC96-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.786537] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=18] start check primary zone(tenant_id=1020) [2024-12-02 13:56:44.787589] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=15][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:44.787669] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=18] start do unit balance [2024-12-02 13:56:44.790136] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=14] start distribute for server status change [2024-12-02 13:56:44.790191] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=18] finish distribute for server status change(ret=0) [2024-12-02 13:56:44.790203] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=12] finish do unit balance(ret=0) [2024-12-02 13:56:44.792525] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=9] start to do tenant group unit balance [2024-12-02 13:56:44.792566] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=17] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:44.792582] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=11] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:44.792602] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=15] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:44.792628] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=12] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:44.792649] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:44.792661] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:44.792672] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=8] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:44.792696] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=17] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:44.792735] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=38] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:44.792747] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=11] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:44.792762] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=11] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:44.792775] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=12] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:44.797807] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF940-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.809695] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DF-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.811431] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.811441] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DF-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.811457] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DF-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.811465] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5DF-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:44.901853] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF941-0-0] [lt=21] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733118999861250, force_dump=false) [2024-12-02 13:56:44.901892] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF941-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.916247] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E0-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:44.917495] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E0-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:44.917505] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:44.917520] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:44.917529] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.005329] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF942-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.022309] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E1-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.023851] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.023861] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.023877] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E1-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.023887] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E1-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.059329] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA2-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:45.076784] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9A-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:45.079943] INFO [RS] inner_dump_statistic_ (ob_disaster_recovery_task_mgr.cpp:1362) [2319637][DRTaskMgr][T0][Y0-0000000000000000-0-0] [lt=19] [DRTASK_NOTICE] disaster recovery task manager statistics(waiting_high_priority_task_cnt=0, executing_high_priority_task_cnt=0, waiting_low_priority_task_cnt=0, executing_low_priority_task_cnt=0) [2024-12-02 13:56:45.108958] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF943-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.128706] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.131526] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.131532] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E2-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.131543] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E2-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.131548] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E2-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.202538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF7-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:45.212403] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF944-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.236332] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E3-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.237540] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.237549] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E3-0-0] [lt=8][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.237564] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.237572] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E3-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.245600] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA6-0-0] [lt=32] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:45.298091] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE9A0-0-0] [lt=20] get number(ret=0, service_type_=0, range=1, base_id=1733119005297560000, start_id=1733119005297560000, end_id=1733119005297560001) [2024-12-02 13:56:45.300076] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE9A0-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119005297560000, v:0}, tenant_id=1002) [2024-12-02 13:56:45.305891] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC88-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:45.308356] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC88-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:45.315902] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF945-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.319363] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF7-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:45.342484] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E4-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.343735] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E4-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.343744] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E4-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.343760] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E4-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.343767] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E4-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.357410] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:45.419726] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF946-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.448665] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E5-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.450539] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E5-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.450546] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E5-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.450559] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E5-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.450567] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E5-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.461365] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADD-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:45.463729] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADD-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:45.487885] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECA9-0-0] [lt=11] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:45.523380] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF947-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.555351] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.556786] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.556796] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E6-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.556811] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E6-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.556820] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.627201] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF948-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.627756] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC88-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:45.630334] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC88-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:45.630355] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC88-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:45.641865] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319549][LeaseQueueTh0][T0][YB42C0A800AF-000628430B4EEAF2-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=14, ddl_arg=NULL) [2024-12-02 13:56:45.661712] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.663197] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E7-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.663206] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.663221] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E7-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.663228] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E7-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.730901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF949-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.739812] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0F-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:45.745177] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0F-0-0] [lt=16] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119005299667003, v:0}}], last_dump_ts_=1733119000685800, force_dump=false) [2024-12-02 13:56:45.745200] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED0F-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:45.768091] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.771681] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E8-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.771688] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E8-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.771703] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E8-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.771711] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E8-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.834927] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.876606] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.879049] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E9-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.879059] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E9-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.879075] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E9-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.879080] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5E9-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:45.938616] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.983932] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EA-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:45.985512] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EA-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:45.985522] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EA-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:45.985537] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EA-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:45.985545] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EA-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.005508] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:46.041938] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94C-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.046234] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF4-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.059404] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA3-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.078140] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9B-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:46.079739] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF8-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.090398] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EB-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.091727] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.091737] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EB-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.091752] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EB-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.091761] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.134377] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:46.135727] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:46.135779] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:46.135790] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:46.135806] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:46.135814] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:46.135825] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:46.135833] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733119011134393, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:46.145178] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94D-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.196652] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.200790] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EC-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.200797] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EC-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.200809] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EC-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.200814] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EC-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.202617] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF8-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.234131] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:46.235262] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=16] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006234069, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:46.235288] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011235286, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:46.245695] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA7-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.248492] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94E-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.305768] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5ED-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.307610] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5ED-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.307621] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5ED-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.307632] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5ED-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.307641] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5ED-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.313746] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC89-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:46.316207] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC89-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:46.319433] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF8-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.328738] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:46.329866] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=15] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006328675, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:46.329890] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011329888, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:46.330989] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=12] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119006329847, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:46.331012] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011331011, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:46.351873] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF94F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.357312] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=7] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119004154300004, replayable_scn:1733119004154300004, readable_scn:1733119004154300004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119006354212, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119004154300004, replayable_scn:1733119004154300004, readable_scn:1733119004154300004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119006354212, ora_rowscn_=1733119005741668001, content_changed=true) [2024-12-02 13:56:46.357554] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:46.358859] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:46.412574] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EE-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.413961] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EE-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.413972] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EE-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.413987] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.413993] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EE-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.455599] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF950-0-0] [lt=22][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.458639] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=16] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119006354212, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119004154300004, replayable_scn:1733119004154300004, readable_scn:1733119004154300004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:46.470508] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADE-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:46.472961] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADE-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:46.487958] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAA-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:46.518912] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.521617] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EF-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.521624] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EF-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.521636] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EF-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.521641] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5EF-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.559663] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF951-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.626552] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F0-0-0] [lt=13] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.633078] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC89-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:46.635655] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC89-0-0] [lt=55] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:46.635676] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC89-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:46.647442] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F0-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.647453] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F0-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.647469] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F0-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.647478] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F0-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.747883] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF952-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.751561] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED10-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:46.752423] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F1-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.753017] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F1-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.753028] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F1-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.753043] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F1-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.753052] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F1-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.758294] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED10-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:46.759918] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE96F-0-0] [lt=19] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:46.802275] INFO [RS.COMPACTION] try_reload (ob_zone_merge_manager.cpp:144) [2325690][T1020_MergeSche][T1020][Y0-0000000000000000-0-0] [lt=55] zone_merge_mgr is already loaded(tenant_id=1020, global_merge_info={tenant_id:1020, cluster:{name:"cluster", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, global_broadcast_scn:{name:"global_broadcast_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, is_merge_error:{name:"is_merge_error", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, error_type:{name:"error_type", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, suspend_merging:{name:"suspend_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}}, zone_merge_infos=[{tenant_id:1020, zone:"zone1", is_merging:{name:"is_merging", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, broadcast_scn:{name:"broadcast_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, last_merged_scn:{name:"last_merged_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, last_merged_time:{name:"last_merged_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, all_merged_scn:{name:"all_merged_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, merge_start_time:{name:"merge_start_time", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, merge_status:{name:"merge_status", is_scn:false, scn:{val:18446744073709551615, v:3}, value:0, need_update:false}, frozen_scn:{name:"frozen_scn", is_scn:true, scn:{val:1, v:0}, value:-1, need_update:false}, start_merge_fail_times:0}]) [2024-12-02 13:56:46.852812] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF953-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.858022] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F2-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.859577] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F2-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.859587] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F2-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.859603] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F2-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.859612] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F2-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.904779] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119006901524, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119006901524, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:46.956349] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF954-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.964451] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F3-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:46.965818] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F3-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:46.965828] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F3-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:46.965843] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F3-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:46.965848] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F3-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:46.998684] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96F-0-0] [lt=31] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:46.999906] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96F-0-0] [lt=18] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:47.001077] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE96F-0-0] [lt=16] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.059479] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA4-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:47.061120] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF955-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.070672] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F4-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.072146] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F4-0-0] [lt=9][errcode=-4018] REACH SYSLOG RATE LIMIT [bandwidth] [2024-12-02 13:56:47.072160] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F4-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.079491] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9C-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:47.105156] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2320160][T1001_BackupDat][T1001][YB42C0A800AF-000628430AAEE96D-0-0] [lt=19] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:47.118344] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:236) [2320159][T1001_BACKUP_SC][T1001][YB42C0A800AF-000628430A9EE94C-0-0] [lt=13] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-12-02 13:56:47.164773] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF956-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.176977] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F5-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.178136] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F5-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.178146] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F5-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.178161] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F5-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.178170] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F5-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.202694] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAF9-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:47.245784] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA8-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:47.268281] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF957-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.282982] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F6-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.284811] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F6-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.284822] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F6-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.284837] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F6-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.284846] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F6-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.319506] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAF9-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:47.321532] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8A-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:47.325115] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8A-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:47.357632] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:47.357649] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=262, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:47.372786] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF958-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.376330] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF958-0-0] [lt=53] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.389616] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F7-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.390933] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F7-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.390944] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F7-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.390960] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F7-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.390968] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F7-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.441195] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=19] start checking lost replicas [2024-12-02 13:56:47.476670] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF959-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.479793] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADF-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:47.482240] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEADF-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:47.488033] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAB-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:47.495734] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F8-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.497260] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F8-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.497271] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F8-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.497286] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F8-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.497295] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F8-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.581776] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.602050] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F9-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.603274] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F9-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.603284] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F9-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.603299] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F9-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.603307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5F9-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.638382] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8A-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:47.640928] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8A-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:47.640948] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8A-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:47.642311] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAF3-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:56:47.685391] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95B-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.708186] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FA-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.710285] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FA-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.710294] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FA-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.710310] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FA-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.710318] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FA-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.765119] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED11-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:47.770614] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED11-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:47.788911] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95C-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.815090] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FB-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.816716] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FB-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.816726] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FB-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.816742] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FB-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.816750] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FB-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.892405] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95D-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.921554] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FC-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:47.923568] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FC-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.923579] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FC-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:47.923595] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FC-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:47.923604] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FC-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:47.969796] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=6] start do ObArchiveSchedulerService round(round=19) [2024-12-02 13:56:47.971983] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=16][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:56:47.972003] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=18][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1 AND dest_no = 0 AND name = 'state', exec_tenant_id=1) [2024-12-02 13:56:47.972068] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=10][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:47.972086] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=17][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:56:47.972098] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2319874][T1_ArchiveSvr][T1][YB42C0A800AF-0006284308FEE95E-0-0] [lt=12][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:47.996104] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:47.999347] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95E-0-0] [lt=52] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.028389] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FD-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.029898] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FD-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.029905] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FD-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.029920] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FD-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.029929] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FD-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.046573] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF5-0-0] [lt=12] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.059552] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA5-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.080847] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9D-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:48.084553] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAF9-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.099730] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF95F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.134745] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FE-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.136080] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FE-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.136090] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FE-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.136106] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FE-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.136114] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FE-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.177877] INFO [RS] process (ob_root_inspection.cpp:494) [2319627][RSInspectTask][T0][Y0-0006284310BEEF90-0-0] [lt=4] purge recyclebin task begin [2024-12-02 13:56:48.177894] INFO [RS] process (ob_root_inspection.cpp:512) [2319627][RSInspectTask][T0][Y0-0006284310BEEF90-0-0] [lt=12] submit purge recyclebin task success(delay=60000000) [2024-12-02 13:56:48.177899] INFO [RS] process (ob_root_inspection.cpp:514) [2319627][RSInspectTask][T0][Y0-0006284310BEEF90-0-0] [lt=5] purge recyclebin task end(delay=60000000) [2024-12-02 13:56:48.202775] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFA-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.203083] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF960-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.240937] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FF-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.242550] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FF-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.242560] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FF-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.242576] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FF-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.242585] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF5FF-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.245877] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECA9-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.306453] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF961-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.319581] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFA-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.330476] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8B-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:48.332969] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8B-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:48.347460] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF600-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.348747] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF600-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.348759] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF600-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.348774] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF600-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.348783] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF600-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.357631] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:48.385028] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE6A-0-0] [lt=8] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:48.410610] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF962-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.453795] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF601-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.462002] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF601-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.462009] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF601-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.462021] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF601-0-0] [lt=9][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.462025] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF601-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.488104] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAC-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:48.488641] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE0-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:48.490967] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE0-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:48.514004] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF963-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.566849] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF602-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.568067] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF602-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.568077] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF602-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.568092] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF602-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.568101] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF602-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.618512] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF964-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.643633] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8B-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:48.646180] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8B-0-0] [lt=54] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:48.646199] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8B-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:48.673045] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF603-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.674753] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF603-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.674764] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF603-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.674778] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF603-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.674787] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF603-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.722140] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF965-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.779106] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED12-0-0] [lt=13] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119008659083000, v:0}}], last_dump_ts_=1733119003717837, force_dump=false) [2024-12-02 13:56:48.779130] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED12-0-0] [lt=20] is primary tenant, no need check readable_scn [2024-12-02 13:56:48.779649] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF604-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.781039] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF604-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.781049] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF604-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.781064] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF604-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.781074] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF604-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.784783] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED12-0-0] [lt=6] is primary tenant, no need check readable_scn [2024-12-02 13:56:48.826075] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF966-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.886272] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF605-0-0] [lt=31] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.888594] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF605-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.888605] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF605-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.888621] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF605-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.888630] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF605-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:48.929458] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF967-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.993599] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF606-0-0] [lt=22] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:48.995424] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF606-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:48.995434] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF606-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:48.995450] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF606-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:48.995459] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF606-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.032711] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF968-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.059623] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA6-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:49.082188] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9E-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:49.100320] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF607-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.101710] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF607-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.101721] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF607-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.101735] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF607-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.101740] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF607-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.136782] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF969-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.202857] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFB-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:49.206696] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF608-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.208237] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF608-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.208248] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF608-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.208263] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF608-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.208272] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF608-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.240212] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96A-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.245966] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAA-0-0] [lt=25] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:49.313177] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF609-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.314495] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF609-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.314505] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF609-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.314521] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF609-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.314530] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF609-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.319655] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFB-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:49.338408] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8C-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:49.340885] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8C-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:49.343759] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96B-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.357413] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119007658158000, replayable_scn:1733119007658158000, readable_scn:1733119007157194004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119009354695, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119007658158000, replayable_scn:1733119007658158000, readable_scn:1733119007157194004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119009354695, ora_rowscn_=1733119008781184000, content_changed=true) [2024-12-02 13:56:49.357648] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=17] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:49.389968] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE9A1-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1733119009389456000, start_id=1733119009389456000, end_id=1733119009389456001) [2024-12-02 13:56:49.392067] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320556][T1_FrzInfoDet][T1][YB42C0A800AF-00062843088EE9A1-0-0] [lt=24] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119009389456000, v:0}, tenant_id=1) [2024-12-02 13:56:49.419447] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.421003] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.421013] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.421029] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60A-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.421039] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60A-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.447344] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.460268] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=15] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119009354695, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119007658158000, replayable_scn:1733119007658158000, readable_scn:1733119007157194004, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:49.488174] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAD-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:49.497627] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE1-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:49.499950] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE1-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:49.526067] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.527444] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.527455] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.527470] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.527476] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60B-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.556410] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96D-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.556627] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96D-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733119004457228, force_dump=false) [2024-12-02 13:56:49.632341] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.633835] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.633846] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.633862] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.633870] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.642764] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAF4-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:49.648924] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8C-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:49.651491] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8C-0-0] [lt=51] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:49.651513] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8C-0-0] [lt=22] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:49.660458] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96E-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.738695] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.740078] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.740088] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60D-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.740107] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60D-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.740112] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60D-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.764367] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF96F-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.790188] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED13-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:49.798129] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED13-0-0] [lt=12] is primary tenant, no need check readable_scn [2024-12-02 13:56:49.845051] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.846836] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.846847] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.846862] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.846871] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.870175] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF970-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.906347] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119009903154, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119009903154, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:49.951965] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60F-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:49.964719] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:49.964730] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:49.964747] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60F-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:49.964752] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF60F-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:49.974082] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF971-0-0] [lt=56] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733119004901852, force_dump=false) [2024-12-02 13:56:49.974126] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF971-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.046854] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF6-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.059701] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA7-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.069680] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF610-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.071278] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF610-0-0] [lt=7][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.071289] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF610-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.071305] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF610-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.071311] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF610-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.077977] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF972-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.083536] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EEC9F-0-0] [lt=16] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:50.089313] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAFA-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.176176] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF611-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.177650] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF611-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.177661] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF611-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.177677] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF611-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.177686] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF611-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.181444] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF973-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.202938] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFC-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.246057] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAB-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.282573] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF612-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.284313] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF612-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.284323] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF612-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.284339] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF612-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.284344] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF612-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.284772] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF974-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.319727] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFC-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.346311] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8D-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:50.348777] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8D-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:50.357703] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:50.388688] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF975-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.389258] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF613-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.390688] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF613-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.390699] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF613-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.390714] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF613-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.390724] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF613-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.488246] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAE-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:50.492010] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF976-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.495622] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF614-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.496185] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF614-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.496195] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF614-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.496211] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF614-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.496219] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF614-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.506736] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE2-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:50.509162] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE2-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:50.534422] INFO [RS] run2 (ob_archive_scheduler_service.cpp:85) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=10] start do ObArchiveSchedulerService round(round=19) [2024-12-02 13:56:50.537796] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=18][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:56:50.537814] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=18][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_log_archive_dest_parameter where tenant_id = 1002 AND dest_no = 0 AND name = 'state', exec_tenant_id=1001) [2024-12-02 13:56:50.537876] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:50.537894] WDIAG [SHARE] get_item (ob_inner_kv_table_operator.cpp:353) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=18][errcode=-4018] failed to get item(ret=-4018, item={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}, need_lock=false) [2024-12-02 13:56:50.537906] WDIAG [SHARE] get_string_value (ob_archive_persist_helper.cpp:410) [2320162][T1001_ArchiveSv][T1001][YB42C0A800AF-000628430ACEE95E-0-0] [lt=11][errcode=-4018] failed to get item(ret=-4018, need_lock=false, item_with_tenant_id={tenant_id:1002, *item:{dest_no:0, name:"state", *value:{value:}}}) [2024-12-02 13:56:50.595785] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF977-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.601048] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF615-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.601630] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF615-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.601641] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF615-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.601660] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF615-0-0] [lt=17][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.601669] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF615-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.654255] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8D-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:50.656840] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8D-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:50.656860] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8D-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:50.699357] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF978-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.706526] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF616-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.708027] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF616-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.708038] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF616-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.708053] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF616-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.708062] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF616-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.802942] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF979-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.803298] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED14-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:50.808273] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED14-0-0] [lt=13] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119010660943000, v:0}}], last_dump_ts_=1733119005745176, force_dump=false) [2024-12-02 13:56:50.808294] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED14-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:50.812946] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF617-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.814121] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF617-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.814131] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF617-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.814148] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF617-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.814156] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF617-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:50.906244] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97A-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.919003] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF618-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:50.920722] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF618-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:50.920732] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF618-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:50.920748] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF618-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:50.920757] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF618-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.008115] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=11] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:51.010063] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97B-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.025618] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF619-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.026900] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF619-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.026910] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF619-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.026926] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF619-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.026934] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF619-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.059782] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA8-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:51.084892] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA0-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:51.113489] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97C-0-0] [lt=31][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.131915] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61A-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.134459] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:51.135850] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:51.135927] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:51.135940] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=13] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:51.135951] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:51.135956] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:51.135964] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=6][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:51.135968] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733119016134476, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:51.143004] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.143015] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.143031] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.143041] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.203015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFD-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:51.218094] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97D-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.235349] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:51.236491] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011235286, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:51.236514] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016236512, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:51.246155] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAC-0-0] [lt=33] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:51.248019] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61B-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.249611] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.249622] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.249638] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.249647] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.302018] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE977-0-0] [lt=21] get number(ret=0, service_type_=0, range=1, base_id=1733119011301859000, start_id=1733119011301859000, end_id=1733119011301859001) [2024-12-02 13:56:51.303500] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2324931][T1019_FrzInfoDe][T1019][YB42C0A800AF-00062843139EE977-0-0] [lt=22] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119011301859000, v:0}, tenant_id=1019) [2024-12-02 13:56:51.319802] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFD-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:51.321658] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97E-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.329952] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:51.331104] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=14] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011329888, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:51.331126] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016331124, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:51.332232] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119011331011, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:51.332253] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=19] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016332251, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:51.354188] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8E-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:51.354575] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.356309] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61C-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.356320] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61C-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.356335] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.356344] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.356696] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8E-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:51.357741] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:51.357758] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=248, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:51.359029] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:51.425189] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF97F-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.461251] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.471840] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61D-0-0] [lt=0] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.488321] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECAF-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:51.515769] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE3-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:51.518170] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE3-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:51.543385] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF980-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.576600] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61E-0-0] [lt=21] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.578485] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.578496] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.578511] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.578520] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.643208] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319550][LeaseQueueTh1][T0][YB42C0A800AF-000628430B4EEAF5-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=15, ddl_arg=NULL) [2024-12-02 13:56:51.646982] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF981-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.659572] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8E-0-0] [lt=24] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:51.662131] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8E-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:51.662151] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8E-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:51.683742] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.685001] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.685012] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.685028] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.685037] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF61F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.750534] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF982-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.789826] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF620-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.802276] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF620-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.802287] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF620-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.802303] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF620-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.802311] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF620-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.813098] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED15-0-0] [lt=5] is primary tenant, no need check readable_scn [2024-12-02 13:56:51.818662] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED15-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:51.853970] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF983-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.907120] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF621-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:51.908757] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF621-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:51.908768] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF621-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:51.908784] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF621-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:51.908793] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF621-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:51.957315] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF984-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.013604] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF622-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.014911] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF622-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.014922] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF622-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.014938] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF622-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.014943] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF622-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.047171] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF7-0-0] [lt=14] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.059857] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECA9-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.060951] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF985-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.086268] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA1-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:52.094022] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAFB-0-0] [lt=19] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.119750] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF623-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.121390] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF623-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.121400] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF623-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.121416] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF623-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.121425] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF623-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.164383] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF986-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.203090] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFE-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.226266] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF624-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.228273] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF624-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.228283] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF624-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.228298] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF624-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.228307] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF624-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.246248] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAD-0-0] [lt=30] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.267927] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF987-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.319875] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.333127] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF625-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.334691] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF625-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.334702] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF625-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.334717] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF625-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.334726] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF625-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.357591] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119010660943000, replayable_scn:1733119010660943000, readable_scn:1733119010660943000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119012355064, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119010660943000, replayable_scn:1733119010660943000, readable_scn:1733119010660943000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119012355064, ora_rowscn_=1733119011815308000, content_changed=true) [2024-12-02 13:56:52.357851] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=21] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:52.362207] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8F-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:52.364690] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC8F-0-0] [lt=26] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:52.373243] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF988-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.439562] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF626-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.441698] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF626-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.441710] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF626-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.441726] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF626-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.441735] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF626-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.461870] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=14] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119012355064, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119010660943000, replayable_scn:1733119010660943000, readable_scn:1733119010660943000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:52.470643] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320118][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE6C-0-0] [lt=10] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:52.476881] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF989-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.488398] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB0-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:52.524894] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE4-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:52.527298] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE4-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:52.546581] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF627-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.548045] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF627-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.548056] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF627-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.548072] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF627-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.548082] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF627-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.580371] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98A-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.652906] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF628-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.654537] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF628-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.654544] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF628-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.654560] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF628-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.654569] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF628-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.664788] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8F-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:52.667274] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8F-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:52.667295] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC8F-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:52.683886] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98B-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.759333] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF629-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.761004] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF629-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.761014] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF629-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.761030] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF629-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.761039] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF629-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.788723] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98C-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.823781] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED16-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:52.829699] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED16-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:52.865984] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.867273] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62A-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.867284] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62A-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.867300] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.867309] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.891959] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98D-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.907933] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=6] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119012905363, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119012905363, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:52.972225] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:52.974090] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62B-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:52.974100] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:52.974116] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62B-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:52.974125] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:52.995426] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98E-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.059937] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAA-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:53.078985] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.080256] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.080267] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.080280] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62C-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.080289] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.081576] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE9A1-0-0] [lt=19] get number(ret=0, service_type_=0, range=1, base_id=1733119013081195000, start_id=1733119013081195000, end_id=1733119013081195001) [2024-12-02 13:56:53.082925] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320658][T1001_FrzInfoDe][T1001][YB42C0A800AF-000628430A7EE9A1-0-0] [lt=23] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119013081195000, v:0}, tenant_id=1001) [2024-12-02 13:56:53.087600] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA2-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:53.098668] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF98F-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.185163] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.186798] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.186810] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62D-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.186826] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.186835] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.201878] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF990-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.203166] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEAFF-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:53.246340] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAE-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:53.291686] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.292982] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.292992] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.293009] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.293018] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.305188] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF991-0-0] [lt=9][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.319944] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEAFF-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:53.357885] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:53.370144] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC90-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:53.372618] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC90-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:53.397879] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.399698] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.399708] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62F-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.399727] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.399736] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF62F-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.408640] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF992-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.488478] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB1-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:53.504644] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF630-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.505997] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF630-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.506007] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF630-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.506022] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF630-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.506031] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF630-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.512200] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF993-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.533868] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE5-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:53.536217] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE5-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:53.610862] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF631-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.612232] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF631-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.612243] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF631-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.612260] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF631-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.612269] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF631-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.615499] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF994-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.643652] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAF6-0-0] [lt=17] [DDL] execute ddl like stmt(ret=0, cost=16, ddl_arg=NULL) [2024-12-02 13:56:53.669955] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC90-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:53.672466] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC90-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:53.672483] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC90-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:53.717147] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF632-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.718885] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF632-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.718895] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF632-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.718912] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF632-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.718921] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF632-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.720077] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF995-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.823764] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF996-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.823919] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF633-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.825285] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF633-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.825295] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF633-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.825311] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF633-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.825320] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF633-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.834623] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED17-0-0] [lt=13] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119013663872000, v:0}}], last_dump_ts_=1733119008779105, force_dump=false) [2024-12-02 13:56:53.834646] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED17-0-0] [lt=18] is primary tenant, no need check readable_scn [2024-12-02 13:56:53.840029] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED17-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:53.873225] INFO [SHARE] get_jobs (ob_backup_data_table_operator.cpp:717) [2324797][T1019_BackupDat][T1019][YB42C0A800AF-0006284313CEE95D-0-0] [lt=18] [DATA_BACKUP]success get jobs(job_attrs=[]) [2024-12-02 13:56:53.930225] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF634-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:53.931161] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF997-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.931780] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF634-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:53.931791] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF634-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:53.931806] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF634-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:53.931815] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF634-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:53.934403] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:131) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=14] [RESTORE] restore scheduler quit [2024-12-02 13:56:53.935581] INFO [SHARE] get_all_recover_table_job (ob_recover_table_persist_helper.cpp:199) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE977-0-0] [lt=13] get targets(ret=0, jobs=[], exec_tenant_id=1, sql=select * from __all_recover_table_job) [2024-12-02 13:56:53.935632] WDIAG [SHARE.SCHEMA] revert (ob_schema_mgr_cache.cpp:122) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE978-0-0] [lt=23][errcode=0] long time to hold one guard(schema_mgr=0xfffea966b8c0, tenant_id=1, version=1733118593816728, cur_timestamp=1733119013935606, ref_timestamp=1733118953931443, lbt()="0x11ab55d8 0x1613a8dc 0x11826844 0x118269d4 0x1159e610 0xa36dff8 0x92af85c 0x1b950c78 0x1b9573f4 0x1b956a10 0x1b954c24 0xffff86b92508 0xffff86bf9cdc") [2024-12-02 13:56:53.935721] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:100) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE979-0-0] [lt=14] [RESTORE] restore scheduler start [2024-12-02 13:56:53.935731] INFO [RS.RESTORE] do_work (ob_restore_scheduler.cpp:107) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=10] [RESTORE] try process restore job [2024-12-02 13:56:53.937183] INFO [COMMON] get_jobs (ob_physical_restore_table_operator.cpp:419) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=9] [RESTORE] get restore jobs(ret=0, job_cnt=1) [2024-12-02 13:56:53.938394] WDIAG [SHARE] parse_one_row_ (ob_inner_table_operator.cpp:954) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=17][errcode=-4018] no row exist(ret=-4018) [2024-12-02 13:56:53.938414] WDIAG [SHARE] do_get_row_ (ob_inner_table_operator.cpp:627) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=19][errcode=-4018] failed to parse row(ret=-4018, sql=select * from __all_restore_job_history where initiator_tenant_id = 1 AND initiator_job_id = 11, exec_tenant_id=1019) [2024-12-02 13:56:53.938487] WDIAG [SHARE] get_row (ob_inner_table_operator.cpp:176) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=11][errcode=-4018] failed to get row(ret=-4018, need_lock=false, key={initiator_tenant_id:1, initiator_job_id:11}) [2024-12-02 13:56:53.938504] WDIAG [SHARE] get_restore_job_history (ob_restore_persist_helper.cpp:827) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=16][errcode=-4018] failed to get persist info(ret=-4018, ret="OB_ENTRY_NOT_EXIST", restore_key={initiator_tenant_id:1, initiator_job_id:11}) [2024-12-02 13:56:53.938515] WDIAG [RS.RESTORE] get_user_restore_job_history (ob_restore_util.cpp:1846) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=10][errcode=-4018] failed to get restore progress(ret=-4018, ret="OB_ENTRY_NOT_EXIST", initiator_job_id=11, initiator_tenant_id=1) [2024-12-02 13:56:53.938525] WDIAG [RS.RESTORE] restore_wait_tenant_finish (ob_restore_scheduler.cpp:1459) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=10][errcode=-4018] failed to get user restore job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", job_info={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:56:53.938557] INFO [RS.RESTORE] process_sys_restore_job (ob_restore_scheduler.cpp:169) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=27] [RESTORE] doing restore(ret=-4018, job={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:56:53.938578] WDIAG [RS.RESTORE] do_work (ob_restore_scheduler.cpp:121) [2319868][T1_REST_SER][T1][YB42C0A800AF-00062843089EE97A-0-0] [lt=20][errcode=-4018] failed to process sys restore job(ret=-4018, ret="OB_ENTRY_NOT_EXIST", job_info={restore_key:{tenant_id:1, job_id:11}, initiator_job_id:0, initiator_tenant_id:1, tenant_id:1020, backup_tenant_id:1004, restore_type:{restore_type:"FULL"}, status:9, comment:"", restore_start_ts:1733118539865219, restoring_start_ts:0, restore_scn:{val:1732874528445031000, v:0}, consistent_scn:{val:1732874437122873000, v:0}, post_data_version:0, source_cluster_version:17180066816, source_data_version:17180066816, restore_option:"pool_list=mq_pool_hy", backup_dest:"file:///home/admin/2024-11-29-18-00-01", description:"", tenant_name:"mysql_hy", pool_list:"mq_pool_hy", locality:"", primary_zone:"", compat_mode:0, compatible:0, kms_info:"", kms_encrypt:false, concurrency:0, passwd_array:"", multi_restore_path_list:{backup_set_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full"], backup_piece_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p11", "file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log/piece_d1013r3p12"], log_path_list:["file:///home/admin/2024-11-29-18-00-01/backup_set_16_full/complement_log"]}, white_list:{table_items:[]}, recover_table:false, using_complement_log:false, backup_compatible:4}) [2024-12-02 13:56:53.939242] INFO [RS] dump_statistics (ob_backup_task_scheduler.cpp:236) [2324796][T1019_BACKUP_SC][T1019][YB42C0A800AF-0006284313BEE94C-0-0] [lt=16] backup scheduler task statistics(waiting_task_cnt=0, executing_task_cnt=0) [2024-12-02 13:56:54.033775] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF998-0-0] [lt=21][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.036670] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF635-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.037234] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF635-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.037245] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF635-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.037261] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF635-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.037270] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF635-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.047492] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF8-0-0] [lt=15] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.060015] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAB-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.088963] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA3-0-0] [lt=19] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:54.098759] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAFC-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.137007] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF999-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.142121] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF636-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.144458] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF636-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.144469] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF636-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.144484] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF636-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.144493] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF636-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.203242] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB00-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.241978] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99A-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.246434] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECAF-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.249349] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF637-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.261813] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF637-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.261822] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF637-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.261837] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF637-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.261845] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF637-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.320002] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB00-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.358000] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:54.360771] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99B-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.366655] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF638-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.368091] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF638-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.368101] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF638-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.368116] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF638-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.368124] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF638-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.377967] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC91-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:54.380437] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC91-0-0] [lt=24] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:54.464373] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99C-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.472850] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF639-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.474285] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF639-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.474293] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF639-0-0] [lt=7][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.474307] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF639-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.474320] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF639-0-0] [lt=11] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.488556] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB2-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.542750] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE6-0-0] [lt=19] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:54.546158] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE6-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:54.568440] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99D-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.568705] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99D-0-0] [lt=55] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874426991810002, v:0}}], last_dump_ts_=1733119009556626, force_dump=false) [2024-12-02 13:56:54.579004] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.580958] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63A-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.580966] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63A-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.580982] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63A-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.580991] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.672083] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99E-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.675242] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC91-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:54.677775] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC91-0-0] [lt=56] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:54.677796] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC91-0-0] [lt=21] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:54.685923] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63B-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.687110] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63B-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.687120] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63B-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.687140] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63B-0-0] [lt=18][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.687150] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63B-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.775399] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF99F-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.792113] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63C-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.792353] INFO [RS.LB] idle (ob_root_balancer.cpp:103) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC97-0-0] [lt=23] root balance idle(idle_time=10000000) [2024-12-02 13:56:54.792377] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2074) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=17] start try disaster recovery [2024-12-02 13:56:54.792390] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] start try tenant disaster recovery(tenant_id=1, only_for_display=false) [2024-12-02 13:56:54.793588] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] start try ls disaster recovery(ls_status_info={tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.794209] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.794220] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63C-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.794236] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63C-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.794242] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63C-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.798026] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=20] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:54.798053] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.798062] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1, ls_id:{id:1}, replicas:[{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.798116] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=51] build ls locality stat map(tenant_id=1, ls_id={id:1}) [2024-12-02 13:56:54.798128] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=9] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.798143] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=13] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118159456434, create_time_us:0, tenant_id:1, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1, in_pool:true, unit:unit_id:1, resource_pool_id:1, unit_group_id:1, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.798178] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.798192] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.798201] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.798209] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.800084] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1, only_for_display=false) [2024-12-02 13:56:54.800101] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=15] start try tenant disaster recovery(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:54.801403] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.805777] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1001, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:54.805804] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.805811] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1001, ls_id:{id:1}, replicas:[{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1001, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.805864] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=50] build ls locality stat map(tenant_id=1001, ls_id={id:1}) [2024-12-02 13:56:54.805877] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.805890] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118160462607, create_time_us:1732871267341550, tenant_id:1001, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.805924] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.805939] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.805949] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.805957] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.807841] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=9] try tenant disaster recovery succ(tenant_id=1001, only_for_display=false) [2024-12-02 13:56:54.809080] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=14] start try tenant disaster recovery(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:54.810303] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=20] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.814661] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:54.814688] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.814699] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1}, replicas:[{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.814750] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=48] build ls locality stat map(tenant_id=1002, ls_id={id:1}) [2024-12-02 13:56:54.814760] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.814772] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167177128, create_time_us:1732871280733688, tenant_id:1002, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.814804] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=25] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.814819] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.814827] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.814833] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=4] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.816594] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] start try ls disaster recovery(ls_status_info={tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.823207] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=17] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1002, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:54.823233] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=21] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.823244] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] try locality alignment(dr_ls_info={resource_tenant_id_:1002, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1002, ls_id:{id:1001}, replicas:[{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.823295] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=48] build ls locality stat map(tenant_id=1002, ls_id={id:1001}) [2024-12-02 13:56:54.823307] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.823320] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=11] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118167179939, create_time_us:1732871299864461, tenant_id:1002, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:3, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1001, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1001, in_pool:true, unit:unit_id:1001, resource_pool_id:1001, unit_group_id:1001, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.823353] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.823368] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.823376] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.823384] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.825102] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1002, only_for_display=false) [2024-12-02 13:56:54.825115] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=12] start try tenant disaster recovery(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:54.826314] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] start try ls disaster recovery(ls_status_info={tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.830553] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=21] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1019, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:54.830580] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.830588] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=5] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1019, ls_id:{id:1}, replicas:[{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1019, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.830638] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=48] build ls locality stat map(tenant_id=1019, ls_id={id:1}) [2024-12-02 13:56:54.830650] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.830663] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118580950623, create_time_us:1733118578521026, tenant_id:1019, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:1, replica_status:"NORMAL", restore_status:{status:0}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.830696] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.830711] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.830720] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.830728] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.832481] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] try tenant disaster recovery succ(tenant_id=1019, only_for_display=false) [2024-12-02 13:56:54.833701] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2149) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=13] start try tenant disaster recovery(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:54.834857] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=17] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.839102] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1}, acc_dr_task=0) [2024-12-02 13:56:54.839130] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=23] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.839138] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1}, replicas:[{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.839189] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=48] build ls locality stat map(tenant_id=1020, ls_id={id:1}) [2024-12-02 13:56:54.839198] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=5] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.839210] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118656791956, create_time_us:1733118591069170, tenant_id:1020, ls_id:{id:1}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.839243] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=26] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.839258] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.839267] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.839274] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.841033] INFO [RS.LB] try_tenant_disaster_recovery (ob_disaster_recovery_worker.cpp:2169) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=4] start try ls disaster recovery(ls_status_info={tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}) [2024-12-02 13:56:54.845749] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED18-0-0] [lt=10] is primary tenant, no need check readable_scn [2024-12-02 13:56:54.847647] INFO [RS.LB] try_remove_permanent_offline_replicas (ob_disaster_recovery_worker.cpp:3666) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=18] finish try remove permanent offline replica(ret=0, ret="OB_SUCCESS", tenant_id=1020, ls_id={id:1001}, acc_dr_task=0) [2024-12-02 13:56:54.847676] INFO [RS.LB] try_replicate_to_unit (ob_disaster_recovery_worker.cpp:3949) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=22] finish try replicate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.847686] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4522) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=10] try locality alignment(dr_ls_info={resource_tenant_id_:1020, zone_locality_array_:[{zone:"zone1", replica_attr_set:{full_replica_attr_array:[{num:1, memstore_percent:100}], logonly_replica_attr_array:[], readonly_replica_attr_array:[], encryption_logonly_replica_attr_array:[], columnstore_replica_attr_array:[]}, zone_set:["zone1"]}], inner_ls_info_:{tenant_id:1020, ls_id:{id:1001}, replicas:[{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}]}, ls_status_info_:{tenant_id:1020, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1010, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, server_stat_array_:[{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}], unit_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], unit_in_group_stat_array_:[{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}], schema_replica_cnt_:1, schema_full_replica_cnt_:1, member_list_cnt_:1, paxos_replica_number_:1, has_leader_:true}, only_for_display=false) [2024-12-02 13:56:54.847739] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:279) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=50] build ls locality stat map(tenant_id=1020, ls_id={id:1001}) [2024-12-02 13:56:54.847750] INFO [RS.LB] build_locality_stat_map (ob_disaster_recovery_worker.cpp:384) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] ls zone locality map info(zone=zone1, zone_replica_desc={replica_desc_array:[cnt:1, {replica_type_:0, memstore_percent_:100, replica_num_:1}], is_readonly_all_server:false, is_columnstore_all_server:false, readonly_memstore_percent:100}) [2024-12-02 13:56:54.847760] INFO [RS.LB] build_replica_stat_map (ob_disaster_recovery_worker.cpp:427) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] ls replica stat(replica_stat_map=[{replica_:{modify_time_us:1733118655283621, create_time_us:1733118641058857, tenant_id:1020, ls_id:{id:1001}, server:"192.168.0.175:2882", sql_port:2881, role:1, member_list:[{server:"192.168.0.175:2882", timestamp:1}], replica_type:0, proposal_id:16, replica_status:"NORMAL", restore_status:{status:6}, property:{memstore_percent_:100}, unit_id:1003, zone:"zone1", paxos_replica_number:1, data_size:0, required_size:0, in_member_list:true, member_time_us:1, learner_list:{learner_num:0, learner_array:[]}, in_learner_list:false, rebuild:false}, server_stat_info_:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}, unit_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}, unit_in_group_stat_info_:{unit_id:1003, in_pool:true, unit:unit_id:1003, resource_pool_id:1004, unit_group_id:1010, zone:"zone1", server:"192.168.0.175:2882", migrate_from_server:"0.0.0.0:0", is_manual_migrate:false, status:0, replica_type:0, server_stat:{server:"192.168.0.175:2882", alive:true, active:true, permanent_offline:false, block:false, stopped:false}}}]) [2024-12-02 13:56:54.847794] INFO [RS.LB] try_locality_alignment (ob_disaster_recovery_worker.cpp:4577) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=27] finish try locality alignment(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.847809] INFO [RS.LB] try_shrink_resource_pools (ob_disaster_recovery_worker.cpp:4721) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=8] finish try shrink resource pool(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.847817] INFO [RS.LB] try_cancel_unit_migration (ob_disaster_recovery_worker.cpp:5449) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try cancel migrate unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.847825] INFO [RS.LB] try_migrate_to_unit (ob_disaster_recovery_worker.cpp:5743) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] finish try migrate to unit(ret=0, ret="OB_SUCCESS", acc_dr_task=0) [2024-12-02 13:56:54.849838] INFO [RS.LB] try_disaster_recovery (ob_disaster_recovery_worker.cpp:2087) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=6] try tenant disaster recovery succ(tenant_id=1020, only_for_display=false) [2024-12-02 13:56:54.849860] INFO [RS] check (ob_migrate_unit_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=18] start check unit migrate finish [2024-12-02 13:56:54.849873] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=5] try check migrate unit finish by tenant(tenant_id=1) [2024-12-02 13:56:54.851454] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED18-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:54.855658] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=7] try check migrate unit finish by tenant(tenant_id=1002) [2024-12-02 13:56:54.865967] INFO [RS] try_check_migrate_unit_finish_by_tenant (ob_migrate_unit_finish_checker.cpp:172) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=20] try check migrate unit finish by tenant(tenant_id=1020) [2024-12-02 13:56:54.875968] INFO [RS] check (ob_alter_locality_finish_checker.cpp:79) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC98-0-0] [lt=19] start to check alter locality finish [2024-12-02 13:56:54.877083] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC99-0-0] [lt=20][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.878225] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC99-0-0] [lt=18][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.879045] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.880524] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9B-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.881652] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9B-0-0] [lt=19][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.883885] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9D-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.884989] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9D-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.886190] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:72) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9D-0-0] [lt=16] start check shrink resource pool [2024-12-02 13:56:54.886212] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9E-0-0] [lt=16] start check shrink resource pool(tenant_id=1) [2024-12-02 13:56:54.887273] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9E-0-0] [lt=9][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.888378] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9E-0-0] [lt=17][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.888451] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9F-0-0] [lt=16] start check shrink resource pool(tenant_id=1002) [2024-12-02 13:56:54.889519] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9F-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.890626] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEEC9F-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.890700] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:93) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA0-0-0] [lt=16] start check shrink resource pool(tenant_id=1020) [2024-12-02 13:56:54.891755] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA0-0-0] [lt=14][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.892857] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA0-0-0] [lt=16][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.892932] INFO [RS] check (ob_shrink_expand_resource_pool_checker.cpp:100) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA0-0-0] [lt=17] finish check shrink resource pool(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:54.892948] INFO [RS] check (ob_alter_primary_zone_checker.cpp:49) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA0-0-0] [lt=14] start check primary zone [2024-12-02 13:56:54.892961] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA1-0-0] [lt=9] start check primary zone(tenant_id=1) [2024-12-02 13:56:54.894019] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA1-0-0] [lt=8][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.894092] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA2-0-0] [lt=16] start check primary zone(tenant_id=1002) [2024-12-02 13:56:54.895142] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA2-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.895216] INFO [RS] check (ob_alter_primary_zone_checker.cpp:75) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=16] start check primary zone(tenant_id=1020) [2024-12-02 13:56:54.896271] WDIAG [RS] find_job (ob_rs_job_table_operator.cpp:334) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=13][errcode=-4008] empty result set(ret=-4008) [2024-12-02 13:56:54.896347] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:179) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=17] start do unit balance [2024-12-02 13:56:54.897317] INFO [SHARE.LOCATION] vtable_nonblock_renew (ob_vtable_location_service.cpp:446) [2319632][RootBalance][T1][YB42C0A800AF-000628430BFEECA3-0-0] [lt=14] add_update_task succeed(ret=0, ret="OB_SUCCESS", task={tenant_id:1, table_id:12274, add_timestamp:1733119014897305}) [2024-12-02 13:56:54.897506] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE6D-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:54.898836] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:363) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=15] start distribute for server status change [2024-12-02 13:56:54.898891] INFO [RS.LB] distribute_for_server_status_change (ob_server_balancer.cpp:414) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=18] finish distribute for server status change(ret=0) [2024-12-02 13:56:54.898902] INFO [RS.LB] balance_servers (ob_server_balancer.cpp:213) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=10] finish do unit balance(ret=0) [2024-12-02 13:56:54.899272] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63D-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:54.900800] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63D-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:54.900810] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:54.900827] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:54.900835] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63D-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:54.901234] INFO [RS.LB] tenant_group_balance (ob_server_balancer.cpp:128) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=8] start to do tenant group unit balance [2024-12-02 13:56:54.901276] INFO [RS.LB] rebalance_servers_v2 (ob_server_balancer.cpp:930) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=17] start to do tenant group unit balance(zone=zone1) [2024-12-02 13:56:54.901291] INFO [RS.LB] generate_zone_server_disk_statistic (ob_server_balancer.cpp:7935) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=10] build zone disk statistic succeed(ret=0, zone_disk_statistic_={zone_:"zone1", server_disk_statistic_array_:[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}], over_disk_waterlevel_:false}) [2024-12-02 13:56:54.901310] INFO [RS.LB] generate_tenant_array (ob_server_balancer.cpp:1084) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=14] generate tenant array finish(ret=0, zone=zone1, standalone_tenant_array=[1001, 1002, 1019, 1020], group_tenant_array=[]) [2024-12-02 13:56:54.901334] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=11] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:54.901354] INFO [RS.LB] generate_available_servers (ob_server_balancer.cpp:1268) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=10] generate available servers finish(ret=0, zone=zone1, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:54.901365] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5311) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=8] start do non-tenantgroup unit balance task(zone=zone1) [2024-12-02 13:56:54.901376] INFO [RS.LB] calc_global_balance_resource_weights (ob_server_balancer.cpp:5640) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=7] finish calc global balance resource weights(ret=0, ret="OB_SUCCESS", zone=zone1, cpu_weights=3.484898771995591504e-01, mem_weights=3.982741453709247037e-01, log_disk_weights=2.532359774295159793e-01, data_disk_weights=0.000000000000000000e+00, available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:54.901399] INFO [RS.LB] divide_complete_server_loads_for_balance (ob_server_balancer.cpp:4547) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=15] divide complete server loads for balance(ret=0, ret="OB_SUCCESS", server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}], available_servers=["192.168.0.175:2882"], wild_server_loads=[], over_server_loads=[], under_server_loads=[{server_:"192.168.0.175:2882", load_sum_:{load_sum_:{unit_config_id:18446744073709551615, name:"", resource:{min_cpu:12, max_cpu:12, memory_size:"12GB", log_disk_size:"109GB", data_disk_size:0, min_iops:9998, max_iops:9998, iops_weight:4, max_net_bandwidth:"-3.7252903e-09GB", net_bandwidth_weight:12, }}}, resource_info_:{cpu_capacity:16, cpu_assigned:12, cpu_assigned_max:12, mem_capacity:"14GB", mem_assigned:"12GB", mem_in_use:0GB, log_disk_capacity:200GB, log_disk_assigned:109GB, log_disk_in_use:5.75GB, data_disk_capacity:10000GB, data_disk_assigned:0GB, data_disk_in_use:0.44140625GB}, inter_ttg_load_value_:"7.407588544869403435e-01"}]) [2024-12-02 13:56:54.901438] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8126) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=39] check server over disk waterlevel(zone=zone1, disk_statistic={server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}, over_disk_waterlevel=false) [2024-12-02 13:56:54.901448] INFO [RS.LB] check_all_available_servers_over_disk_waterlevel (ob_server_balancer.cpp:8132) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=10] check all available servers over disk waterlevel(zone=zone1, all_available_servers_over_disk_waterlevel=false, zone_server_over_disk_waterlevel=false, server_disk_statistic_array=[{server_:"192.168.0.175:2882", disk_in_use_:473956352, disk_total_:10737418240000, wild_server_:false}]) [2024-12-02 13:56:54.901463] INFO [RS.LB] make_available_servers_balance (ob_server_balancer.cpp:5293) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=10] finish zone servers balance(ret=0, ret="OB_SUCCESS", disk_waterlevel=8.000000000000000444e-01, balance_task_count=0, balance_reason="load_balance", all_available_servers_disk_over=false, zone_over_disk_waterlevel=false, standalone_units.count()=2, not_grant_units.count()=0) [2024-12-02 13:56:54.901483] INFO [RS.LB] do_non_tenantgroup_unit_balance_task (ob_server_balancer.cpp:5348) [2319632][RootBalance][T0][YB42C0A800AF-000628430BFEECA3-0-0] [lt=19] finish do non-tenantgroup unit balance task(ret=0, ret="OB_SUCCESS", zone=zone1, upper_lmt=7.777967972112873607e-01, wild_server_loads.count()=0, over_server_loads.count()=0, under_server_loads.count()=1, standalone_units=[{unit_id:1001, tenant_id:1002, resource_pool_id:1001, unit_config:{unit_config_id:1001, name:"ocp_meta_unit", resource:{min_cpu:1, max_cpu:1, memory_size:"2GB", log_disk_size:"6GB", data_disk_size:0, min_iops:9223372036854775807, max_iops:9223372036854775807, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:1, }}, unit:0xfffe9c877ce8, unit_config:0xfffe9c865d18, pool:0xfffe9c883688}, {unit_id:1003, tenant_id:1020, resource_pool_id:1004, unit_config:{unit_config_id:1003, name:"hy_unit_config", resource:{min_cpu:8, max_cpu:8, memory_size:"8GB", log_disk_size:"100GB", data_disk_size:0, min_iops:10000, max_iops:10000, iops_weight:1, max_net_bandwidth:INT64_MAX, net_bandwidth_weight:8, }}, unit:0xfffe9c877c10, unit_config:0xfffe9c865c38, pool:0xfffe9c8832d0}], not_grant_units=[], available_servers=["192.168.0.175:2882"]) [2024-12-02 13:56:54.982657] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A1-0-0] [lt=57] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1732874408973130000, v:0}}], last_dump_ts_=1733119009974081, force_dump=false) [2024-12-02 13:56:54.982694] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A1-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.005896] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.007113] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63E-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.007123] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.007139] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.007148] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.060092] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAC-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:55.085877] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A2-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.090297] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA4-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:55.112002] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.113565] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63F-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.113572] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63F-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.113588] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.113598] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF63F-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.189147] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A3-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.203315] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB01-0-0] [lt=12] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:55.218454] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF640-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.219657] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF640-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.219663] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF640-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.219675] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF640-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.219683] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF640-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.246534] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECB0-0-0] [lt=36] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:55.292411] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A4-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.306017] INFO [STORAGE.TRANS] get_number (ob_id_service.cpp:393) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE9A1-0-0] [lt=22] get number(ret=0, service_type_=0, range=1, base_id=1733119015305032000, start_id=1733119015305032000, end_id=1733119015305032001) [2024-12-02 13:56:55.320080] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB01-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:55.320525] INFO [RS] renew_snapshot_gc_scn (ob_major_merge_info_manager.cpp:354) [2320687][T1002_FrzInfoDe][T1002][YB42C0A800AF-000628430B6EE9A1-0-0] [lt=21] renew snapshot_gc_scn(ret=0, new_snapshot_gc_scn={val:1733119015305032000, v:0}, tenant_id=1002) [2024-12-02 13:56:55.324576] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF641-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.326322] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF641-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.326332] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF641-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.326349] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF641-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.326358] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF641-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.357830] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119013663872000, replayable_scn:1733119013663872000, readable_scn:1733119013663872000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119015354485, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119013663872000, replayable_scn:1733119013663872000, readable_scn:1733119013663872000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119015354485, ora_rowscn_=1733119014848592000, content_changed=true) [2024-12-02 13:56:55.358073] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:55.358089] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=12] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=236, return_code_array=[0], broadcast_times=1) [2024-12-02 13:56:55.385767] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC92-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:55.388242] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC92-0-0] [lt=27] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:55.395868] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A5-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.431263] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF642-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.432726] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF642-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.432736] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF642-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.432753] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF642-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.432758] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF642-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.463516] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=17] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119015354485, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119013663872000, replayable_scn:1733119013663872000, readable_scn:1733119013663872000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:55.488631] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB3-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:55.499322] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.537759] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF643-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.539577] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF643-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.539588] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF643-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.539604] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF643-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.539613] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF643-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.553050] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE7-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:55.555495] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE7-0-0] [lt=23] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:55.602734] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A7-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.644087] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAF7-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=18, ddl_arg=NULL) [2024-12-02 13:56:55.644494] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF644-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.645712] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF644-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.645719] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF644-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.645732] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF644-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.645741] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF644-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.680466] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC92-0-0] [lt=19] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:55.682988] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC92-0-0] [lt=57] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:55.683007] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC92-0-0] [lt=20] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:55.712006] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.750540] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF645-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.751729] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF645-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.751739] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF645-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.751755] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF645-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.751764] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF645-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.815405] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9A9-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.856592] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF646-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.856662] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED19-0-0] [lt=13] is primary tenant, no need check readable_scn [2024-12-02 13:56:55.857979] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF646-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.857986] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF646-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.858002] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF646-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.858011] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF646-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:55.863055] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED19-0-0] [lt=12] ls readable scn in memory(ls_id={id:1001}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119015666027000, v:0}}], last_dump_ts_=1733119010808272, force_dump=false) [2024-12-02 13:56:55.863075] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED19-0-0] [lt=17] is primary tenant, no need check readable_scn [2024-12-02 13:56:55.909598] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=5] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119015907011, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119015907011, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:55.919764] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AA-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.962814] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF647-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:55.964388] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF647-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:55.964395] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF647-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:55.964407] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF647-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:55.964415] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF647-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.010875] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1020, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:56.031525] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AB-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.047817] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAF9-0-0] [lt=17] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.060171] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAD-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.069294] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF648-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.070552] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF648-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.070562] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF648-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.070576] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF648-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.070584] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF648-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.091631] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA5-0-0] [lt=17] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:56.103477] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAFD-0-0] [lt=20] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.134546] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:56.134922] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AC-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.135944] WDIAG [RS] get_dbms_job_infos_in_tenant (ob_dbms_job_utils.cpp:363) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=7][errcode=-5019] execute query failed(ret=-5019, sql=select * from __all_job, tenant_id=1020) [2024-12-02 13:56:56.135999] WDIAG [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:613) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=-5019] fail to exec job_utils_.get_dbms_job_infos_in_tenant(tenant_id, allocator, job_infos)(ret=-5019) [2024-12-02 13:56:56.136009] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10] load and register new jobs(ret=-5019, job_key=NULL, job_key=NULL, job_infos=[]) [2024-12-02 13:56:56.136021] WDIAG [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:590) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=10][errcode=-5019] fail to exec load_and_register_new_jobs(tenant_ids.at(i))(ret=-5019) [2024-12-02 13:56:56.136025] INFO [RS] load_and_register_all_jobs (ob_dbms_job_master.cpp:593) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=4] register single tenant(ret=-5019, tenant_ids.at(i)=1020, tenant_schema->get_compatibility_mode()=0) [2024-12-02 13:56:56.136037] WDIAG [RS] scheduler_job (ob_dbms_job_master.cpp:368) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=9][errcode=-5019] fail to exec load_and_register_all_jobs(job_key)(ret=-5019) [2024-12-02 13:56:56.136045] WDIAG [RS] scheduler (ob_dbms_job_master.cpp:340) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=8][errcode=0] fail to scheduler single dbms job(ret=0, tmp_ret=-5019, job_key={check_job:false, check_new:false, check_new_tenant:true, execute_at:1733119021134564, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:56.175463] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF649-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.177001] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF649-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.177011] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF649-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.177026] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF649-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.177034] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF649-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.203391] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB02-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.236574] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:56.237752] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=18] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016236512, delay:5000000, job_id:0, tenant_id:1019}, job_key=0xfffeb4af6120, job_infos=[]) [2024-12-02 13:56:56.237779] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=24] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119021237777, delay:5000000, job_id:0, tenant_id:1019}) [2024-12-02 13:56:56.238957] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AD-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.246625] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECB1-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.281932] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.283300] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64A-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.283311] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64A-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.283326] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64A-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.283334] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64A-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.320156] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB02-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.331187] INFO [RS] scheduler (ob_dbms_job_master.cpp:329) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] dbms job master wait timeout, no entry(ret=-4018) [2024-12-02 13:56:56.332338] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=13] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016331124, delay:5000000, job_id:0, tenant_id:1001}, job_key=0xfffeb4af60a0, job_infos=[]) [2024-12-02 13:56:56.332362] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=21] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119021332360, delay:5000000, job_id:0, tenant_id:1001}) [2024-12-02 13:56:56.333484] INFO [RS] load_and_register_new_jobs (ob_dbms_job_master.cpp:614) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=11] load and register new jobs(ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119016332251, delay:5000000, job_id:0, tenant_id:1002}, job_key=0xfffeb4af60e0, job_infos=[]) [2024-12-02 13:56:56.333506] INFO [RS] scheduler (ob_dbms_job_master.cpp:342) [2319636][DBMS_JOB_MASTER][T0][YB42C0A800AF-00062843037EE950-0-0] [lt=20] success to scheduler single dbms job(ret=0, tmp_ret=0, job_key={check_job:false, check_new:true, check_new_tenant:false, execute_at:1733119021333505, delay:5000000, job_id:0, tenant_id:1002}) [2024-12-02 13:56:56.342624] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AE-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.358079] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=5] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:56.359386] INFO [RS] refresh_service_name (ob_tenant_info_loader.cpp:1028) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh service_names(ret=0, ret="OB_SUCCESS", tenant_id_=1002, epoch_=0, all_service_names_=[]) [2024-12-02 13:56:56.388227] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64B-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.389759] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64B-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.389770] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64B-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.389785] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.389793] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64B-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.393669] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC93-0-0] [lt=23] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:56.396146] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC93-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:56.445919] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9AF-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.488711] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB4-0-0] [lt=21] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:56.494579] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.495794] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64C-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.495804] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.495819] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64C-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.495826] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64C-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.529977] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE6E-0-0] [lt=20] [DDL] execute ddl like stmt(ret=0, cost=11, ddl_arg=NULL) [2024-12-02 13:56:56.534990] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE6F-0-0] [lt=21] [DDL] execute ddl like stmt(ret=0, cost=5, ddl_arg=NULL) [2024-12-02 13:56:56.549641] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B0-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.562151] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE8-0-0] [lt=20] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:56.564519] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE8-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:56.600771] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64D-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.602650] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64D-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.602657] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64D-0-0] [lt=6][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.602673] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.602681] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64D-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.653152] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B1-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.685683] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC93-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:56.688207] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC93-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:56.688225] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC93-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:56.707545] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64E-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.708888] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64E-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.708899] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.708915] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64E-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.708924] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.756400] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B2-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.760006] INFO [RS.COMPACTION] can_start_work (ob_freeze_info_detector.cpp:265) [2325689][T1020_FrzInfoDe][T1020][YB42C0A800AF-00062843179EE970-0-0] [lt=23] tenant is in abnormal status, no need detect now(tenant_id=1020, tenant_schema={tenant_id:1020, schema_version:1733118578035224, tenant_name:"mysql_hy", name_case_mode:-1, read_only:false, primary_zone:"RANDOM", locality:"FULL{1}@zone1", previous_locality:"", compatibility_mode:0, gmt_modified:1733118578035236, drop_tenant_time:-1, status:3, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:56.813854] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.815740] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64F-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.815750] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64F-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.815764] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64F-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.815770] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF64F-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.859703] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B3-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.873078] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1A-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:56:56.880086] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1A-0-0] [lt=8] is primary tenant, no need check readable_scn [2024-12-02 13:56:56.920665] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF650-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:56.922175] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF650-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:56.922185] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF650-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:56.922199] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF650-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:56.922207] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF650-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:56.971565] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B4-0-0] [lt=20][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.005265] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE970-0-0] [lt=28] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:57.006470] INFO [RS] do_ls_balance_task_ (ob_recovery_ls_service.cpp:1308) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE970-0-0] [lt=15] no need process balance task(tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}) [2024-12-02 13:56:57.007641] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324582][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317BEE970-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=0, ret="OB_SUCCESS", idle_time_us=10000000, start_scn={val:18446744073709551615, v:3}, thread_idx=0, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.027192] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF651-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.028464] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF651-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.028473] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF651-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.028488] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF651-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.028496] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF651-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.060250] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAE-0-0] [lt=18] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:57.088765] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B5-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.092974] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA6-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:57.133414] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF652-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.134942] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF652-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.134949] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF652-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.134961] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF652-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.134966] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF652-0-0] [lt=4] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.192081] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B6-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.203463] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB03-0-0] [lt=13] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:57.239929] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF653-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.241399] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF653-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.241410] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF653-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.241425] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF653-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.241434] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF653-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.246715] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECB2-0-0] [lt=29] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:57.295393] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B7-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.320232] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB03-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:57.346394] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF654-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.348103] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF654-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.348114] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF654-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.348130] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF654-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.348139] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF654-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.358129] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=6] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:57.399286] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.401497] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC94-0-0] [lt=22] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:57.402709] INFO [RS] update_replayable_point_from_tenant_info_ (ob_ls_recovery_reportor.cpp:464) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B8-0-0] [lt=50] update_replayable_point_from_tenant_info_ success(ret=0, ret="OB_SUCCESS", replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.405018] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC94-0-0] [lt=20] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:57.444974] INFO [RS] check_lost_replicas (ob_lost_replica_checker.cpp:117) [2319631][LostRepCheck][T0][Y0-0000000000000000-0-0] [lt=17] start checking lost replicas [2024-12-02 13:56:57.453053] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF655-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.454293] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF655-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.454303] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF655-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.454320] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF655-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.454329] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF655-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.488790] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB5-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:57.503045] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9B9-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.559338] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF656-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.560821] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF656-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.560833] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF656-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.560848] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF656-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.560857] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF656-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.571225] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE9-0-0] [lt=21] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:57.573624] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAE9-0-0] [lt=22] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:57.606351] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BA-0-0] [lt=11][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.644509] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2319551][LeaseQueueTh2][T0][YB42C0A800AF-000628430B4EEAF8-0-0] [lt=18] [DDL] execute ddl like stmt(ret=0, cost=13, ddl_arg=NULL) [2024-12-02 13:56:57.665799] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF657-0-0] [lt=19] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.666985] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF657-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.666995] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF657-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.667010] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF657-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.667018] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF657-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.690907] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC94-0-0] [lt=25] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:57.693435] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC94-0-0] [lt=52] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:57.693454] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC94-0-0] [lt=18] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:57.710994] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BB-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.771980] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF658-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.773443] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF658-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.773453] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF658-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.773474] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF658-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.773484] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF658-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.814366] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BC-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.878495] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF659-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.879832] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF659-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.879842] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF659-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.879857] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF659-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.879865] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF659-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:57.899885] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1B-0-0] [lt=11] is primary tenant, no need check readable_scn [2024-12-02 13:56:57.905161] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1B-0-0] [lt=15] is primary tenant, no need check readable_scn [2024-12-02 13:56:57.917958] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BD-0-0] [lt=12][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.984716] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65A-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:57.985893] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65A-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:57.985903] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65A-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:57.985915] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65A-0-0] [lt=10][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:57.985923] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65A-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.021586] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BE-0-0] [lt=13][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.024472] INFO [RS] update_ls_recovery (ob_ls_recovery_reportor.cpp:345) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BE-0-0] [lt=50] tenant update ls recovery stat(ret=0, ret="OB_SUCCESS", ls_recovery_stat={tenant_id:1020, ls_id:{id:1001}, sync_scn:{val:1732874426991810003, v:0}, readable_scn:{val:1732874426991810002, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.048117] INFO [RS] send_heartbeat_ (ob_heartbeat_service.cpp:276) [2319897][T1_HBService][T1][YB42C0A800AF-0006284308BEEAFA-0-0] [lt=16] send_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.060328] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECAF-0-0] [lt=19] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.090809] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65B-0-0] [lt=16] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.092037] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65B-0-0] [lt=5][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.092044] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65B-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.092059] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65B-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.092067] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65B-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.094328] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA7-0-0] [lt=18] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:58.108321] INFO [RS] manage_heartbeat_ (ob_heartbeat_service.cpp:408) [2319898][T1_HBService][T1][YB42C0A800AF-0006284308AEEAFE-0-0] [lt=22] manage_heartbeat_ has finished one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.124871] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9BF-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.143284] INFO [RS] get_dbms_sched_job_infos_in_tenant (ob_dbms_sched_table_operator.cpp:677) [2320057][T1_DBMSSched][T1][Y0-0000000000000000-0-0] [lt=18] failed to get result(ret=-4008) [2024-12-02 13:56:58.196919] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65C-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.198520] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65C-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.198531] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65C-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.198546] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65C-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.198555] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65C-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.203538] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB04-0-0] [lt=15] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.228901] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C0-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.246806] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECB3-0-0] [lt=28] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.286324] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320099][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE74-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=10, ddl_arg=NULL) [2024-12-02 13:56:58.289031] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320124][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE75-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:56:58.289334] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320103][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE76-0-0] [lt=15] [DDL] execute ddl like stmt(ret=0, cost=6, ddl_arg=NULL) [2024-12-02 13:56:58.289978] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE77-0-0] [lt=19] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:58.290179] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320105][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE78-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=3, ddl_arg=NULL) [2024-12-02 13:56:58.290350] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320115][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE79-0-0] [lt=16] [DDL] execute ddl like stmt(ret=0, cost=9, ddl_arg=NULL) [2024-12-02 13:56:58.290802] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320108][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE7C-0-0] [lt=9] [DDL] execute ddl like stmt(ret=0, cost=8, ddl_arg=NULL) [2024-12-02 13:56:58.291615] INFO [RS] process_ (ob_rs_rpc_processor.h:226) [2320098][T1_L0_G0][T1][YB42C0A800AF-000628430E2EEE80-0-0] [lt=14] [DDL] execute ddl like stmt(ret=0, cost=7, ddl_arg=NULL) [2024-12-02 13:56:58.303807] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65D-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.305249] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65D-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.305262] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65D-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.305278] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65D-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.305288] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65D-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.320306] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB04-0-0] [lt=17] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.332445] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C1-0-0] [lt=15][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.357982] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119016819263000, replayable_scn:1733119016819263000, readable_scn:1733119016666660000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, new_refresh_time_us=1733119018354428, tenant_id=1002, tenant_info_={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119016819263000, replayable_scn:1733119016819263000, readable_scn:1733119016666660000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}, last_sql_update_time_=1733119018354428, ora_rowscn_=1733119017902193000, content_changed=true) [2024-12-02 13:56:58.358241] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=22] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:58.410309] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65E-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.410389] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC95-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1001, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:58.411659] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65E-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.411670] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65E-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.411686] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65E-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.411695] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65E-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.412873] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2320152][T1001_COMMONLSS][T1001][YB42C0A800AF-000628430D2EEC95-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:58.435907] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C2-0-0] [lt=14][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.465173] INFO [RS] get_valid_sts_after (ob_tenant_info_loader.cpp:460) [2320413][T1002_STSWorker][T1002][Y0-0000000000000000-0-0] [lt=18] get_valid_sts_after(ret=-4076, ret="OB_NEED_WAIT", specified_time_us=0, last_sql_update_time=1733119018354428, tenant_info={tenant_id:1002, tenant_role:"PRIMARY", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1733119016819263000, replayable_scn:1733119016819263000, readable_scn:1733119016666660000, recovery_until_scn:4611686018427387903, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"NORMAL"}}) [2024-12-02 13:56:58.488869] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2319861][T1_StandbySchem][T1][YB42C0A800AF-0006284303EEECB6-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:58.516667] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65F-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.534145] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65F-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.534156] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65F-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.534172] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65F-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.534182] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF65F-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.539243] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C3-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.580259] INFO [RS] gather_stat (ob_ls_service_helper.cpp:1243) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAEA-0-0] [lt=18] [LS_MGR] gather stat(ret=0, ret="OB_SUCCESS", primary_zone_=["zone1"], unit_group_array_=[{unit_group_id:1010, unit_status:0, ls_group_ids:[1001]}]) [2024-12-02 13:56:58.582623] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2324789][T1019_COMMONLSS][T1019][YB42C0A800AF-0006284316AEEAEA-0-0] [lt=25] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:58.639230] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF660-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.640614] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF660-0-0] [lt=6][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.640625] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF660-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.640641] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF660-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.640650] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF660-0-0] [lt=8] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.643341] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C4-0-0] [lt=17][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.696079] INFO [RS] process_all_ls (ob_primary_ls_service.cpp:136) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC95-0-0] [lt=34] [PRIMARY_LS_SERVICE] finish process tenant(ret=0, ret="OB_SUCCESS", tenant_id=1002, task_cnt=0, machine_array=[{ls_id:{id:1}, status_info:{tenant_id:1002, ls_id:{id:1}, ls_group_id:0, status:"NORMAL", unit_group_id:0, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1}, ls_group_id:0, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1, v:0}}}, {ls_id:{id:1001}, status_info:{tenant_id:1002, ls_id:{id:1001}, ls_group_id:1001, status:"NORMAL", unit_group_id:1001, primary_zone:"zone1", flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}}, ls_info:{id:{id:1001}, ls_group_id:1001, flag:{flag:0, is_duplicate:false, is_block_tablet_in:false}, status:2, operation_type_:1, create_scn:{val:1732871292059216000, v:0}}}], tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:58.698553] INFO [RS] process_all_ls_status_to_steady_ (ob_primary_ls_service.cpp:466) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC95-0-0] [lt=53] [PRIMARY_LS_SERVICE] finish process all ls status to steady(ret=0, ret="OB_SUCCESS", tenant_id_=1002) [2024-12-02 13:56:58.698571] INFO [RS] do_work (ob_primary_ls_service.cpp:95) [2320318][T1002_PLSSer][T1002][YB42C0A800AF-000628430D7EEC95-0-0] [lt=17] [PRIMARY_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tenant_schema={tenant_id:1002, schema_version:1732871292217168, tenant_name:"ocp_meta", zone_list:[cnt:1, "zone1"], primary_zone:"RANDOM", charset_type:0, locked:false, comment:"", name_case_mode:-1, read_only:false, locality_str:"FULL{1}@zone1", zone_replica_attr_array:[cnt:1, {replica_attr_set:{full_replica_attr_array:[cnt:1, {num:1, memstore_percent:100}], logonly_replica_attr_array:[cnt:0], readonly_replica_attr_array:[cnt:0], encryption_logonly_replica_attr_array:[cnt:0], columnstore_replica_attr_array:[cnt:0]}, zone_set:[cnt:1, "zone1"], zone:"zone1"}], primary_zone_array:[], previous_locality_str:"", default_tablegroup_id:18446744073709551615, default_tablegroup_name:"", compatibility_mode:0, drop_tenant_time:-1, status:0, in_recyclebin:false, arbitration_service_status:{status:3}}) [2024-12-02 13:56:58.745709] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF661-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.746770] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C5-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.747126] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF661-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.747136] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF661-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.747152] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF661-0-0] [lt=12][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.747160] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF661-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.849559] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C6-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.852220] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF662-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.853299] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF662-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.853305] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF662-0-0] [lt=5][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.853320] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF662-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.853328] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF662-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:58.909944] INFO [RS] dump_all_replica_readable_scn_ (ob_ls_recovery_stat_handler.cpp:792) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1C-0-0] [lt=14] ls readable scn in memory(ls_id={id:1}, replicas_scn_=[{server:"192.168.0.175:2882", readable_scn:{val:1733119018668933001, v:0}}], last_dump_ts_=1733119013834622, force_dump=false) [2024-12-02 13:56:58.909971] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1C-0-0] [lt=22] is primary tenant, no need check readable_scn [2024-12-02 13:56:58.911302] INFO [RS] refresh_tenant_info (ob_tenant_info_loader.cpp:791) [2324575][T1020_TenantInf][T1020][Y0-0000000000000000-0-0] [lt=9] refresh tenant info(ret=0, ret="OB_SUCCESS", new_tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, new_refresh_time_us=1733119018908787, tenant_id=1020, tenant_info_={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, last_sql_update_time_=1733119018908787, ora_rowscn_=1733118656880408000, content_changed=false) [2024-12-02 13:56:58.915338] INFO [RS] skip_check_member_list_change_ (ob_ls_recovery_stat_handler.cpp:112) [2320316][T1002_TeRec][T1002][YB42C0A800AF-00062843066EED1C-0-0] [lt=21] is primary tenant, no need check readable_scn [2024-12-02 13:56:58.953405] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C7-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.958519] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF663-0-0] [lt=30] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:58.959994] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF663-0-0] [lt=8][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:58.960004] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF663-0-0] [lt=9][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:58.960019] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF663-0-0] [lt=13][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:58.960027] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF663-0-0] [lt=7] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:59.057028] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C8-0-0] [lt=18][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.060403] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320315][T1002_StandbySc][T1002][YB42C0A800AF-00062843067EECB0-0-0] [lt=16] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:59.064994] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF664-0-0] [lt=17] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:59.066229] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF664-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.066240] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF664-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:59.066256] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF664-0-0] [lt=14][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:59.066265] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF664-0-0] [lt=9] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:59.095683] INFO [RS] do_work (ob_common_ls_service.cpp:138) [2319862][T1_COMMONLSSe][T1][YB42C0A800AF-00062843091EECA8-0-0] [lt=21] [COMMON_LS_SERVICE] finish one round(ret=0, ret="OB_SUCCESS", tmp_ret=0, tmp_ret="OB_SUCCESS", idle_time_us=1000000) [2024-12-02 13:56:59.160579] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9C9-0-0] [lt=19][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.171205] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF665-0-0] [lt=18] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:59.191963] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF665-0-0] [lt=10][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.191975] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF665-0-0] [lt=11][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:59.191988] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF665-0-0] [lt=11][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:59.191993] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF665-0-0] [lt=5] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:59.203615] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324577][T1020_StandbySc][T1020][YB42C0A800AF-0006284311BEEB05-0-0] [lt=14] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:59.246898] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2320151][T1001_StandbySc][T1001][YB42C0A800AF-0006284304FEECB4-0-0] [lt=27] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:59.292407] WDIAG [RS] update_ls_recovery_stat_ (ob_ls_recovery_reportor.cpp:265) [2324578][T1020_TeRec][T1020][YB42C0A800AF-0006284311CEF9CA-0-0] [lt=16][errcode=0] failed to update_ls_replayable_point(tmp_ret=-4002, tmp_ret="OB_INVALID_ARGUMENT", ls={running_state:{state:1}, ls_meta:{tenant_id:1020, ls_id:{id:1001}, ls_persistent_state:{state:4}, clog_checkpoint_scn:{val:1732874426991810003, v:0}, clog_base_lsn:{lsn:6978895872}, rebuild_seq:1, migration_status:0, gc_state_:1, offline_scn_:{val:18446744073709551615, v:3}, restore_status:{status:6}, replayable_point:{val:1732874425490194000, v:0}, tablet_change_checkpoint_scn:{val:1732874426991810002, v:0}, all_id_meta:{id_meta:[{limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}, {limited_id:1, latest_log_ts:{val:18446744073709551615, v:3}}]}, transfer_scn:{val:0, v:0}, rebuild_info:{status:{status:0}, type:{type:0}, tablet_id_array:{id_array:[]}, src:[type = 0], [address = ]}, transfer_meta_info:{src_ls:{id:0}, src_scn:{val:1, v:0}, trans_status:0, tablet_id_array:{id_array:[]}, data_version:1(0, 0, 1)}, store_format:{store_type:1, store_type_str:"NORMAL"}}, switch_epoch:1, log_handler:{role:2, proposal_id:16, palf_env_:0xfffe781de030, is_in_stop_state_:false, is_inited_:true, id_:1001}, restore_handler:{is_inited:true, is_in_stop_state:false, id:1001, proposal_id:16, role:1, parent:0xffff66897900, context:{issue_task_num:1, issue_version:1733118655309045735, last_fetch_ts:-1, max_submit_lsn:{lsn:7046000640}, max_fetch_lsn:{lsn:18446744073709551615}, max_fetch_scn:{val:18446744073709551615, v:3}, error_context:{ret_code:0, trace_id:Y0-0000000000000000-0-0, error_type:1110564574, err_lsn:{lsn:18446744073709551615}}, task_count:0}, restore_context:{seek_done:false, lsn:{lsn:18446744073709551615}}}, is_inited:true, tablet_gc_handler:{tablet_persist_trigger:0, is_inited:true}, startup_transfer_info:{ls_id:{id:0}, transfer_start_scn:{val:18446744073709551615, v:3}}, need_delay_resource_recycle:false}, replayable_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.296883] INFO [RS] report_sys_ls_recovery_stat_in_trans_ (ob_recovery_ls_service.cpp:1225) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF666-0-0] [lt=14] report sys ls recovery stat(ret=0, ret="OB_SUCCESS", sync_scn={val:1732874420916288000, v:0}, only_update_readable_scn=true, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}, comment="report readable_scn while start_scn is invalid") [2024-12-02 13:56:59.297431] WDIAG [RS] seek_log_iterator_ (ob_recovery_ls_service.cpp:292) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF666-0-0] [lt=9][errcode=-4018] locate_by_scn_coarsely failed(ret=-4018, ret="OB_ENTRY_NOT_EXIST", sync_scn={val:1732874420916288000, v:0}) [2024-12-02 13:56:59.297443] WDIAG [RS] process_thread1_ (ob_recovery_ls_service.cpp:226) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF666-0-0] [lt=10][errcode=-4018] failed to seek log iterator(ret=-4018, ret="OB_ENTRY_NOT_EXIST", ls_recovery_stat={tenant_id:1020, ls_id:{id:1}, sync_scn:{val:1732874420916288000, v:0}, readable_scn:{val:1732874408973130000, v:0}, create_scn:{val:1, v:0}, drop_scn:{val:1, v:0}, config_version_:{proposal_id:16, config_seq:2}}) [2024-12-02 13:56:59.297459] WDIAG [RS] do_work (ob_recovery_ls_service.cpp:163) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF666-0-0] [lt=15][errcode=-4018] failed to process thread1(ret=-4018, ret="OB_ENTRY_NOT_EXIST") [2024-12-02 13:56:59.297475] INFO [RS] do_work (ob_recovery_ls_service.cpp:167) [2324583][T1020_RecLSSer][T1020][YB42C0A800AF-0006284317AEF666-0-0] [lt=15] [LS_RECOVERY] finish one round(ret=-4018, ret="OB_ENTRY_NOT_EXIST", idle_time_us=100000, start_scn={val:18446744073709551615, v:3}, thread_idx=1, tenant_info={tenant_id:1020, tenant_role:"RESTORE", switchover_status:"NORMAL", switchover_epoch:0, sync_scn:1732874420916288000, replayable_scn:1732874420916288000, readable_scn:1732874408973130000, recovery_until_scn:1732874437122873000, log_mode:"NOARCHIVELOG", max_ls_id:1001, restore_data_mode:{restore data mode:"REMOTE"}}, restore_status={ls_id:-1, sync_lsn:-1, sync_scn:{val:18446744073709551615, v:3}, sync_status:0, err_code:0, comment:}) [2024-12-02 13:56:59.320384] INFO [RS] do_work (ob_standby_schema_refresh_trigger.cpp:70) [2324788][T1019_StandbySc][T1019][YB42C0A800AF-00062843125EEB05-0-0] [lt=20] [STANDBY_SCHEMA_REFRESH_TRIGGER] finish one round(ret=0, ret="OB_SUCCESS") [2024-12-02 13:56:59.358263] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:413) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=10] refresh tenant info content success(ret=0, ret="OB_SUCCESS", i=0, dest="192.168.0.175:2882") [2024-12-02 13:56:59.358280] INFO [RS] broadcast_tenant_info_content_ (ob_tenant_info_loader.cpp:422) [2320313][T1002_TenantInf][T1002][Y0-0000000000000000-0-0] [lt=13] broadcast_tenant_info_content_ finished(ret=0, ret="OB_SUCCESS", tenant_id=1002, cost_time_us=242, return_code_array=[0], broadcast_times=1)